Merge pull request #270 from networktocode/release-v2.4.0

Release v2.4.0

Author: chadell

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## v2.4.0 - 2024-02-20
+
+### Added
+
+- [#260](https://github.com/networktocode/circuit-maintenance-parser/pull/260) - Add Google parser
+- [#259](https://github.com/networktocode/circuit-maintenance-parser/pull/259) - Add Crown Castle fiber parser
+- [#258](https://github.com/networktocode/circuit-maintenance-parser/pull/258) - Add Netflix parser
+
+### Changed
+
+- [#264](https://github.com/networktocode/circuit-maintenance-parser/pull/264) - Adopt Pydantic 2.0
+- [#256](https://github.com/networktocode/circuit-maintenance-parser/pull/256) - Improved Equinix parser
+
+### Fixed
+
+- [#257](https://github.com/networktocode/circuit-maintenance-parser/pull/257) - Update incorrect file comment
+- [#255](https://github.com/networktocode/circuit-maintenance-parser/pull/255) -
+  Properly process Amazon emergency maintenance notifications
+
 ## v2.3.0 - 2023-12-15
 
 ### Added

README.md

@@ -24,7 +24,7 @@ You can leverage this library in your automation framework to process circuit ma
 - **provider**: identifies the provider of the service that is the subject of the maintenance notification.
 - **account**: identifies an account associated with the service that is the subject of the maintenance notification.
 - **maintenance_id**: contains text that uniquely identifies (at least within the context of a specific provider) the maintenance that is the subject of the notification.
-- **circuits**: list of circuits affected by the maintenance notification and their specific impact. Note that in a maintenance canceled notification, some providers omit the circuit list, so this may be blank for maintenance notifications with a status of CANCELLED.
+- **circuits**: list of circuits affected by the maintenance notification and their specific impact. Note that in a maintenance canceled or completed notification, some providers omit the circuit list, so this may be blank for maintenance notifications with a status of CANCELLED or COMPLETED.
 - **start**: timestamp that defines the starting date/time of the maintenance in GMT.
 - **end**: timestamp that defines the ending date/time of the maintenance in GMT.
 - **stamp**: timestamp that defines the update date/time of the maintenance in GMT.
@@ -71,12 +71,15 @@ By default, there is a `GenericProvider` that supports a `SimpleProcessor` using
 - BSO
 - Cogent
 - Colt
+- Crown Castle Fiber
 - Equinix
 - EXA (formerly GTT)
 - HGC
+- Google
 - Lumen
 - Megaport
 - Momentum
+- Netflix (AS2906 only)
 - Seaborn
 - Sparkle
 - Telstra

circuit_maintenance_parser/__init__.py

@@ -12,13 +12,16 @@
     BSO,
     Cogent,
     Colt,
+    CrownCastle,
     Equinix,
     EUNetworks,
     GTT,
+    Google,
     HGC,
     Lumen,
     Megaport,
     Momentum,
+    Netflix,
     NTT,
     PacketFabric,
     Seaborn,
@@ -38,13 +41,16 @@
     BSO,
     Cogent,
     Colt,
+    CrownCastle,
     Equinix,
     EUNetworks,
+    Google,
     GTT,
     HGC,
     Lumen,
     Megaport,
     Momentum,
+    Netflix,
     NTT,
     PacketFabric,
     Seaborn,
@@ -80,7 +86,6 @@ def get_provider_class(provider_name: str) -> Type[GenericProvider]:
         if provider_parser.get_provider_type() == provider_name:
             break
     else:
-
         raise NonexistentProviderError(
             f"{provider_name} is not a currently supported provider. Only {', '.join(SUPPORTED_PROVIDER_NAMES)}"
         )
@@ -90,7 +95,6 @@ def get_provider_class(provider_name: str) -> Type[GenericProvider]:
 
 def get_provider_class_from_sender(email_sender: str) -> Type[GenericProvider]:
     """Returns the notification parser class for an email sender address."""
-
     for provider_parser in SUPPORTED_PROVIDERS:
         if provider_parser.get_default_organizer() == email_sender:
             break

circuit_maintenance_parser/data.py

@@ -3,7 +3,7 @@
 from typing import List, NamedTuple, Optional, Type, Set
 
 import email
-from pydantic import BaseModel, Extra
+from pydantic import BaseModel
 from circuit_maintenance_parser.constants import EMAIL_HEADER_SUBJECT, EMAIL_HEADER_DATE
 
 
@@ -18,7 +18,7 @@ class DataPart(NamedTuple):
     content: bytes
 
 
-class NotificationData(BaseModel, extra=Extra.forbid):
+class NotificationData(BaseModel, extra="forbid"):
     """Base class for Notification Data types."""
 
     data_parts: List[DataPart] = []

circuit_maintenance_parser/output.py

@@ -8,7 +8,7 @@
 
 from typing import List
 
-from pydantic import BaseModel, validator, StrictStr, StrictInt, Extra, PrivateAttr
+from pydantic import field_validator, BaseModel, StrictStr, StrictInt, PrivateAttr
 
 
 class Impact(str, Enum):
@@ -52,7 +52,7 @@ class Status(str, Enum):
     NO_CHANGE = "NO-CHANGE"
 
 
-class CircuitImpact(BaseModel, extra=Extra.forbid):
+class CircuitImpact(BaseModel, extra="forbid"):
     """CircuitImpact class.
 
     Each Circuit Maintenance can contain multiple affected circuits, and each one can have a different level of impact.
@@ -73,23 +73,31 @@ class CircuitImpact(BaseModel, extra=Extra.forbid):
         ... )
         Traceback (most recent call last):
         ...
-        pydantic.error_wrappers.ValidationError: 1 validation error for CircuitImpact
+        pydantic_core._pydantic_core.ValidationError: 1 validation error for CircuitImpact
         impact
-          value is not a valid enumeration member; permitted: 'NO-IMPACT', 'REDUCED-REDUNDANCY', 'DEGRADED', 'OUTAGE' (type=type_error.enum; enum_values=[<Impact.NO_IMPACT: 'NO-IMPACT'>, <Impact.REDUCED_REDUNDANCY: 'REDUCED-REDUNDANCY'>, <Impact.DEGRADED: 'DEGRADED'>, <Impact.OUTAGE: 'OUTAGE'>])
+          Input should be 'NO-IMPACT', 'REDUCED-REDUNDANCY', 'DEGRADED' or 'OUTAGE' [type=enum, input_value='wrong impact', input_type=str]
     """
 
     circuit_id: StrictStr
     # Optional Attributes
     impact: Impact = Impact.OUTAGE
 
     # pylint: disable=no-self-argument
-    @validator("impact")
+    @field_validator("impact")
+    @classmethod
     def validate_impact_type(cls, value):
         """Validate Impact type."""
         if value not in Impact:
             raise ValueError("Not a valid impact type")
         return value
 
+    def to_json(self):
+        """Return a JSON serializable dict."""
+        return {
+            "circuit_id": self.circuit_id,
+            "impact": self.impact.value,
+        }
+
 
 class Metadata(BaseModel):
     """Metadata class to provide context about the Maintenance object."""
@@ -100,15 +108,16 @@ class Metadata(BaseModel):
     generated_by_llm: bool = False
 
 
-class Maintenance(BaseModel, extra=Extra.forbid):
+class Maintenance(BaseModel, extra="forbid"):
     """Maintenance class.
 
     Mandatory attributes:
         provider: identifies the provider of the service that is the subject of the maintenance notification
         account:  identifies an account associated with the service that is the subject of the maintenance notification
         maintenance_id:  contains text that uniquely identifies the maintenance that is the subject of the notification
         circuits: list of circuits affected by the maintenance notification and their specific impact. Note this can be
-            an empty list for notifications with a CANCELLED status if the provider does not populate the circuit list.
+            an empty list for notifications with a CANCELLED or COMPLETED status if the provider does not populate the
+            circuit list.
         status: defines the overall status or confirmation for the maintenance
         start: timestamp that defines the start date of the maintenance in GMT
         end: timestamp that defines the end date of the maintenance in GMT
@@ -163,34 +172,40 @@ class Maintenance(BaseModel, extra=Extra.forbid):
 
     def __init__(self, **data):
         """Initialize the Maintenance object."""
-        self._metadata = data.pop("_metadata")
+        metadata = data.pop("_metadata")
         super().__init__(**data)
+        self._metadata = metadata
 
-    # pylint: disable=no-self-argument
-    @validator("status")
+    @field_validator("status")
+    @classmethod
     def validate_status_type(cls, value):
         """Validate Status type."""
         if value not in Status:
             raise ValueError("Not a valid status type")
         return value
 
-    @validator("provider", "account", "maintenance_id", "organizer")
+    @field_validator("provider", "account", "maintenance_id", "organizer")
+    @classmethod
     def validate_empty_strings(cls, value):
         """Validate emptry strings."""
         if value in ["", "None"]:
             raise ValueError("String is empty or 'None'")
         return value
 
-    @validator("circuits")
+    @field_validator("circuits")
+    @classmethod
     def validate_empty_circuits(cls, value, values):
         """Validate non-cancel notifications have a populated circuit list."""
-        if len(value) < 1 and values["status"] != "CANCELLED":
+        values = values.data
+        if len(value) < 1 and str(values["status"]) in ("CANCELLED", "COMPLETED"):
             raise ValueError("At least one circuit has to be included in the maintenance")
         return value
 
-    @validator("end")
+    @field_validator("end")
+    @classmethod
     def validate_end_time(cls, end, values):
         """Validate that End time happens after Start time."""
+        values = values.data
         if "start" not in values:
             raise ValueError("Start time is a mandatory attribute.")
         start = values["start"]
@@ -208,6 +223,6 @@ def to_json(self) -> str:
         return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=2)
 
     @property
-    def metadata(self):
+    def metadata(self) -> Metadata:
         """Get Maintenance Metadata."""
         return self._metadata

circuit_maintenance_parser/parser.py

@@ -12,7 +12,7 @@
 import bs4  # type: ignore
 from bs4.element import ResultSet  # type: ignore
 
-from pydantic import BaseModel
+from pydantic import BaseModel, PrivateAttr
 from icalendar import Calendar  # type: ignore
 
 from circuit_maintenance_parser.errors import ParserError
@@ -34,15 +34,15 @@ class Parser(BaseModel):
     """
 
     # _data_types are used to match the Parser to to each type of DataPart
-    _data_types = ["text/plain", "plain"]
+    _data_types = PrivateAttr(["text/plain", "plain"])
 
     # TODO: move it to where it is used, Cogent parser
     _geolocator = Geolocator()
 
     @classmethod
     def get_data_types(cls) -> List[str]:
         """Return the expected data type."""
-        return cls._data_types
+        return cls._data_types.get_default()
 
     @classmethod
     def get_name(cls) -> str:
@@ -92,7 +92,7 @@ class ICal(Parser):
     Reference: https://tools.ietf.org/html/draft-gunter-calext-maintenance-notifications-00
     """
 
-    _data_types = ["text/calendar", "ical", "icalendar"]
+    _data_types = PrivateAttr(["text/calendar", "ical", "icalendar"])
 
     def parser_hook(self, raw: bytes, content_type: str):
         """Execute parsing."""
@@ -164,7 +164,7 @@ def parse_ical(gcal: Calendar) -> List[Dict]:
 class Html(Parser):
     """Html parser."""
 
-    _data_types = ["text/html", "html"]
+    _data_types = PrivateAttr(["text/html", "html"])
 
     @staticmethod
     def remove_hex_characters(string):
@@ -201,7 +201,11 @@ def clean_line(line):
 class EmailDateParser(Parser):
     """Parser for Email Date."""
 
-    _data_types = [EMAIL_HEADER_DATE]
+    _data_types = PrivateAttr(
+        [
+            EMAIL_HEADER_DATE,
+        ]
+    )
 
     def parser_hook(self, raw: bytes, content_type: str):
         """Execute parsing."""
@@ -214,7 +218,11 @@ def parser_hook(self, raw: bytes, content_type: str):
 class EmailSubjectParser(Parser):
     """Parse data from subject or email."""
 
-    _data_types = [EMAIL_HEADER_SUBJECT]
+    _data_types = PrivateAttr(
+        [
+            EMAIL_HEADER_SUBJECT,
+        ]
+    )
 
     def parser_hook(self, raw: bytes, content_type: str):
         """Execute parsing."""
@@ -236,7 +244,7 @@ def bytes_to_string(string):
 class Csv(Parser):
     """Csv parser."""
 
-    _data_types = ["application/csv", "text/csv", "application/octet-stream"]
+    _data_types = PrivateAttr(["application/csv", "text/csv", "application/octet-stream"])
 
     def parser_hook(self, raw: bytes, content_type: str):
         """Execute parsing."""
@@ -255,7 +263,11 @@ def parse_csv(raw: bytes) -> List[Dict]:
 class Text(Parser):
     """Text parser."""
 
-    _data_types = ["text/plain"]
+    _data_types = PrivateAttr(
+        [
+            "text/plain",
+        ]
+    )
 
     def parser_hook(self, raw: bytes, content_type: str):
         """Execute parsing."""
@@ -278,7 +290,7 @@ def parse_text(self, text) -> List[Dict]:
 class LLM(Parser):
     """LLM parser."""
 
-    _data_types = ["text/html", "html", "text/plain"]
+    _data_types = PrivateAttr(["text/html", "html", "text/plain"])
 
     _llm_question = """Please, could you extract a JSON form without any other comment,
     with the following JSON schema (timestamps in EPOCH and taking into account the GMT offset):

circuit_maintenance_parser/parsers/aws.py

@@ -1,4 +1,4 @@
-"""AquaComms parser."""
+"""AWS parser."""
 import hashlib
 import logging
 import quopri
@@ -65,7 +65,7 @@ def parse_text(self, text):
         maintenace_id = ""
         status = Status.CONFIRMED
         for line in text.splitlines():
-            if "planned maintenance" in line.lower():
+            if "planned maintenance" in line.lower() or "maintenance has been scheduled" in line.lower():
                 data["summary"] = line
                 search = re.search(
                     r"([A-Z][a-z]{2}, [0-9]{1,2} [A-Z][a-z]{2,9} [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{2,3}) to ([A-Z][a-z]{2}, [0-9]{1,2} [A-Z][a-z]{2,9} [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{2,3})",