only fetch json schema and licenses once (#126)

* only fetch json schema and licenses once #125

* fix error

Author: fqrious

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "txt2detection"
-version = "1.0.3"
+version = "1.0.4"
 authors = [
   { name = "dogesec" }
 ]

tests/src/test_utils.py

@@ -23,7 +23,7 @@ def test_as_date_with_datetime_and_date():
     assert as_date(d_obj) == d_obj
 
 def test_validate_token_count():
-    provider = parse_model('openai')
+    provider = parse_model('anthropic')
     with patch.object(type(provider), 'count_tokens') as mock_count_tokens:
         mock_count_tokens.return_value = 1025
         with pytest.raises(Exception):

txt2detection/models.py

@@ -3,9 +3,10 @@
 import re
 import typing
 import uuid
+import requests
 from slugify import slugify
 from datetime import date as dt_date
-from typing import Any, List, Literal, Optional, Union
+from typing import Any, ClassVar, List, Literal, Optional, Union
 from uuid import UUID
 
 import jsonschema
@@ -194,6 +195,7 @@ class BaseDetection(BaseModel):
     level: Level
     _custom_id = None
     _extra_data: dict
+    sigma_json_schema: ClassVar = requests.get("https://github.com/SigmaHQ/sigma-specification/raw/refs/heads/main/json-schema/sigma-detection-rule-schema.json").json()
 
     def model_post_init(self, __context):
         self.tags = self.tags or []
@@ -249,9 +251,7 @@ def make_rule(self, bundler: "Bundler"):
     def validate_rule_with_json_schema(self, rule):
         jsonschema.validate(
             rule,
-            {
-                "$ref": "https://github.com/SigmaHQ/sigma-specification/raw/refs/heads/main/json-schema/sigma-detection-rule-schema.json"
-            },
+            self.sigma_json_schema,
         )
 
     @property

txt2detection/utils.py

@@ -55,11 +55,13 @@ def validate_token_count(max_tokens, input, extractor: BaseAIExtractor):
     if  token_count > max_tokens:
         raise Exception(f"{extractor.extractor_name}: input_file token count ({token_count}) exceeds INPUT_TOKEN_LIMIT ({max_tokens})")
 
+
+@lru_cache(maxsize=5)
+def get_licenses(date):
+    resp = requests.get("https://github.com/spdx/license-list-data/raw/refs/heads/main/json/licenses.json")
+    return {l['licenseId']: l['name'] for l in resp.json()['licenses']}
+
 def valid_licenses():
-    @lru_cache(maxsize=5)
-    def get_licenses(date):
-        resp = requests.get("https://github.com/spdx/license-list-data/raw/refs/heads/main/json/licenses.json")
-        return {l['licenseId']: l['name'] for l in resp.json()['licenses']}
     return get_licenses(datetime.now().date().isoformat())