diff --git a/environment.yml b/environment.yml index 5536214b5..be6597fd3 100644 --- a/environment.yml +++ b/environment.yml @@ -30,12 +30,12 @@ dependencies: - numpy>=1.23.4,<2.0.0 - oauthlib>=3.2.2,<4.0.0 - cryptography>=38.0.3 - - azure-identity>=1.12.0,<2.0.0 + - azure-identity>=1.20.0,<2.0.0 - azure-storage-file-datalake>=12.12.0,<13.0.0 - azure-keyvault-secrets>=4.7.0,<5.0.0 - azure-mgmt-storage>=21.0.0 - boto3>=1.28.2,<2.0.0 - - pyodbc>=4.0.39,<5.3.0 + - pyodbc>=5.2.0,<6.0.0 - fastapi>=0.115.6,<1.0.0 - httpx>=0.24.1,<1.0.0 - pyspark>=3.3.0,<3.6.0 @@ -44,7 +44,7 @@ dependencies: - grpcio-status>=1.48.1 - googleapis-common-protos>=1.56.4 - openjdk>=11.0.15,<12.0.0 - - openai>=1.13.3,<2.0.0 + - openai>=1.59.0,<2.0.0 - mkdocs-material==9.5.20 - mkdocs-material-extensions==1.3.1 - mkdocstrings==0.25.0 @@ -53,12 +53,12 @@ dependencies: - mkdocs-autorefs>=1.0.0,<1.1.0 - pygments==2.16.1 - pymdown-extensions==10.8.1 - - databricks-sql-connector>=3.1.0,<4.0.0 + - databricks-sql-connector>=3.6.0,<3.7.0 - semver>=3.0.0,<4.0.0 - xlrd>=2.0.1 - pygithub>=1.59.0 - - pydantic>=2.6.0,<3.0.0 - - pyjwt>=2.8.0,<3.0.0 + - pydantic>=2.10.0,<3.0.0 + - pyjwt>=2.10.0,<3.0.0 - twine==4.0.2 - delta-sharing-python>=1.0.0,<2.0.0 - polars>=0.18.8,<1.0.0 @@ -70,21 +70,23 @@ dependencies: - great-expectations>=0.18.8,<1.0.0 - statsmodels>=0.14.1,<0.15.0 - pmdarima>=2.0.4 - - protobuf>=4.25.0,<5.0.0 + - protobuf>=5.28.2,<5.29.0 - pip: - databricks-sdk>=0.59.0,<1.0.0 - dependency-injector>=4.41.0,<5.0.0 - - azure-functions>=1.15.0,<2.0.0 + - azure-functions>=1.20.0,<2.0.0 - azure-mgmt-eventgrid>=10.2.0 - - hvac>=1.1.1 - - langchain>=0.2.0,<0.3.0 - - langchain-community>=0.2.0,<0.3.0 + - hvac>=2.3.0 + - langchain>=0.3.27,<0.4.0 + - langchain-community>=0.3.27,<0.4.0 + - langchain-core>=0.3.28,<0.4.0 + - langchain-text-splitters>=0.3.5,<0.4.0 - build==0.10.0 - deltalake>=0.10.1,<1.0.0 - trio>=0.22.1 - sqlparams>=5.1.0,<6.0.0 - entsoe-py>=0.5.10,<1.0.0 - - web3>=6.18.0,<7.0.0 - - eth-typing>=4.2.3,<5.0.0 - - pandas<3.0.0 + - web3>=7.7.0,<8.0.0 + - eth-typing>=5.0.1,<6.0.0 + - pandas>=2.0.1,<2.3.0 - moto[s3]>=5.0.16,<6.0.0 diff --git a/setup.py b/setup.py index bff07fd8b..91a1abe2b 100644 --- a/setup.py +++ b/setup.py @@ -28,10 +28,10 @@ long_description = (here / "PYPI-README.md").read_text() INSTALL_REQUIRES = [ - "databricks-sql-connector>=3.1.0,<4.0.0", + "databricks-sql-connector>=3.1.0,<3.7.0", "pyarrow>=14.0.1,<17.0.0", "azure-identity>=1.12.0,<2.0.0", - "pandas<3.0.0", + "pandas>=2.0.1,<2.3.0", "jinja2>=3.1.5,<4.0.0", "importlib_metadata>=7.0.0,<8.0.0", "semver>=3.0.0,<4.0.0", @@ -40,12 +40,15 @@ "grpcio-status>=1.48.1", "googleapis-common-protos>=1.56.4", "pydantic>=2.6.0,<3.0.0", + "protobuf>=5.28.2,<5.29.0", ] LANGCHAIN_PACKAGES = [ - "langchain>=0.2.0,<0.3.0", - "langchain-community>=0.2.0,<0.3.0", - "openai>=1.13.3,<2.0.0", + "langchain>=0.3.27,<0.4.0", + "langchain-community>=0.3.27,<0.4.0", + "langchain-core>=0.3.28,<0.4.0", + "langchain-text-splitters>=0.3.5,<0.4.0", + "openai>=1.59.0,<2.0.0", "statsmodels>=0.14.1,<0.15.0", "pmdarima>=2.0.4", ] @@ -62,9 +65,10 @@ "azure-mgmt-storage>=21.0.0", "azure-mgmt-eventgrid>=10.2.0", "boto3>=1.28.2,<2.0.0", - "hvac>=1.1.1", + "hvac>=2.3.0", "azure-keyvault-secrets>=4.7.0,<5.0.0", - "web3>=6.18.0,<7.0.0", + "web3>=7.7.0,<8.0.0", + "eth-typing>=5.0.1,<6.0.0", "polars[deltalake]>=0.18.8,<1.0.0", "delta-sharing>=1.0.0,<1.1.0", "xarray>=2023.1.0,<2023.8.0", diff --git a/src/api/requirements.txt b/src/api/requirements.txt index 1e455e720..0a1201e3a 100644 --- a/src/api/requirements.txt +++ b/src/api/requirements.txt @@ -1,24 +1,27 @@ # Do not include azure-functions-worker as it may conflict with the Azure Functions platform -azure-functions==1.18.0 +azure-functions==1.20.0 fastapi==0.115.6 -pydantic==2.6.0 +pydantic==2.10.0 # turbodbc==4.11.0 -pyodbc==4.0.39 -importlib_metadata>=1.0.0 -databricks-sql-connector==2.9.3 -azure-identity==1.17.0 +pyodbc==5.2.0 +importlib_metadata>=7.0.0 +databricks-sql-connector==3.6.0 +azure-identity==1.20.0 oauthlib>=3.2.2 -pandas>=2.0.1,<3.0.0 +pandas>=2.0.1,<2.3.0 numpy==1.26.4 jinja2==3.1.5 -pytz==2024.1 +pytz==2024.2 semver==3.0.2 xlrd==2.0.1 -packaging==23.2 +packaging==24.2 grpcio>=1.48.1 grpcio-status>=1.48.1 googleapis-common-protos>=1.56.4 -langchain>=0.2.0,<0.3.0 -langchain-community>=0.2.0,<0.3.0 -openai==1.13.3 -pyjwt==2.8.0 +protobuf>=5.28.2,<5.29.0 +langchain>=0.3.27,<0.4.0 +langchain-community>=0.3.27,<0.4.0 +langchain-core>=0.3.28,<0.4.0 +langchain-text-splitters>=0.3.5,<0.4.0 +openai>=1.59.0,<2.0.0 +pyjwt==2.10.0 diff --git a/src/api/v1/models.py b/src/api/v1/models.py index 8255bc280..000a517a5 100644 --- a/src/api/v1/models.py +++ b/src/api/v1/models.py @@ -350,23 +350,9 @@ class TagsBodyParams(BaseModel): class PlotQueryParams: def __init__( self, - sample_rate: str = Query( - ..., - description="sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead.", - examples=[5], - deprecated=True, - ), - sample_unit: str = Query( - ..., - description="sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead.", - examples=["second", "minute", "hour", "day"], - deprecated=True, - ), time_interval_rate: str = DuplicatedQueryParameters.time_interval_rate, time_interval_unit: str = DuplicatedQueryParameters.time_interval_unit, ): - self.sample_rate = sample_rate - self.sample_unit = sample_unit self.time_interval_rate = time_interval_rate self.time_interval_unit = time_interval_unit @@ -374,18 +360,6 @@ def __init__( class ResampleQueryParams: def __init__( self, - sample_rate: str = Query( - ..., - description="sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead.", - examples=[5], - deprecated=True, - ), - sample_unit: str = Query( - ..., - description="sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead.", - examples=["second", "minute", "hour", "day"], - deprecated=True, - ), time_interval_rate: str = DuplicatedQueryParameters.time_interval_rate, time_interval_unit: str = DuplicatedQueryParameters.time_interval_unit, agg_method: str = Query( @@ -394,8 +368,6 @@ def __init__( examples=["first", "last", "avg", "min", "max"], ), ): - self.sample_rate = sample_rate - self.sample_unit = sample_unit self.time_interval_rate = time_interval_rate self.time_interval_unit = time_interval_unit self.agg_method = agg_method @@ -431,23 +403,9 @@ def __init__( class InterpolateQueryParams: def __init__( self, - sample_rate: str = Query( - ..., - description="sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead.", - examples=[5], - deprecated=True, - ), - sample_unit: str = Query( - ..., - description="sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead.", - examples=["second", "minute", "hour", "day"], - deprecated=True, - ), time_interval_rate: str = DuplicatedQueryParameters.time_interval_rate, time_interval_unit: str = DuplicatedQueryParameters.time_interval_unit, ): - self.sample_rate = sample_rate - self.sample_unit = sample_unit self.time_interval_rate = time_interval_rate self.time_interval_unit = time_interval_unit @@ -462,8 +420,8 @@ def __init__( ), timestamps: List[Union[date, datetime]] = Query( ..., - description="Timestamps in format YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss or YYYY-MM-DDTHH:mm:ss+zz:zz", - examples=[EXAMPLE_DATE, EXAMPLE_DATETIME, EXAMPLE_DATETIME_TIMEZOME], + description="Timestamps in format YYYY-MM-DDTHH:mm:ss or YYYY-MM-DDTHH:mm:ss+zz:zz", + examples=[EXAMPLE_DATETIME, EXAMPLE_DATETIME_TIMEZOME], ), window_length: int = Query( ..., description="Window Length in days", examples=[1] diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py index 8f31a96e1..3797e5877 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py @@ -471,7 +471,9 @@ def _build_output_query(sql_query_list, to_json, limit, offset): output_sql_query = " ".join( [ output_sql_query, - "to_json(struct(*), map('timestampFormat', 'yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXXX')) AS Value", + "to_json(struct(*), map('timestampFormat', " + "'yyyy-MM-dd\\'T\\'HH:mm:ss.SSSSSSSSSXXX'" + ")) AS Value", ] ) else: @@ -1251,7 +1253,7 @@ def _interpolation_at_time(parameters_dict: dict) -> str: 'SELECT {% if to_json is defined and to_json == true %}to_json(struct(p.`EventTime`, p.`TagName`, p.`Value`, m.`UoM`), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}p.`EventTime`, p.`TagName`, p.`Value`, m.`UoM`{% endif %} FROM project p ' "LEFT OUTER JOIN " "{% if metadata_source is defined and metadata_source is not none %}" - "`{{ metadata_source|lower }}` m ON p.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " + "{{ metadata_source|lower }} m ON p.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " "{% else %}" "`{{ business_unit|lower }}`.`sensors`.`{{ asset|lower }}_{{ data_security_level|lower }}_metadata` m ON p.`{{ tagname_column }}` = m.`{{ tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " "{% endif %}" @@ -1378,7 +1380,7 @@ def _latest_query(parameters_dict: dict) -> str: 'SELECT {% if to_json is defined and to_json == true %}to_json(struct(l.*, m.`UoM), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}l.*, m.`UoM`{% endif %} FROM latest l ' "LEFT OUTER JOIN " "{% if metadata_source is defined and metadata_source is not none %}" - "`{{ metadata_source|lower }}` m ON l.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` " + "{{ metadata_source|lower }} m ON l.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` " "{% else %}" "`{{ business_unit|lower }}`.`sensors`.`{{ asset|lower }}_{{ data_security_level|lower }}_metadata` m ON l.`{{ tagname_column }}` = m.`{{ tagname_column }}` " "{% endif %}" @@ -1454,7 +1456,7 @@ def _time_weighted_average_query(parameters_dict: dict) -> str: ",fill_step AS (SELECT f.*, IFNULL(m.Step, false) AS Step FROM fill_value f " "LEFT JOIN " "{% if metadata_source is defined and metadata_source is not none %}" - "`{{ metadata_source|lower }}` m ON f.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}`) " + "{{ metadata_source|lower }} m ON f.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}`) " "{% else %}" "`{{ business_unit|lower }}`.`sensors`.`{{ asset|lower }}_{{ data_security_level|lower }}_metadata` m ON f.`{{ tagname_column }}` = m.`{{ tagname_column }}`) " "{% endif %}" @@ -1612,7 +1614,7 @@ def _circular_stats_query(parameters_dict: dict) -> str: 'SELECT {% if to_json is defined and to_json == true %}to_json(struct(p.*, m.`UoM`), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}p.*, m.`UoM`{% endif %} FROM project p ' "LEFT OUTER JOIN " "{% if metadata_source is defined and metadata_source is not none %}" - "`{{ metadata_source|lower }}` m ON p.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " + "{{ metadata_source|lower }} m ON p.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " "{% else %}" "`{{ business_unit|lower }}`.`sensors`.`{{ asset|lower }}_{{ data_security_level|lower }}_metadata` m ON p.`{{ tagname_column }}` = m.`{{ tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " "{% endif %}" @@ -1650,7 +1652,7 @@ def _circular_stats_query(parameters_dict: dict) -> str: 'SELECT {% if to_json is defined and to_json == true %}to_json(struct(p.*, m.`UoM`), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}p.*, m.`UoM`{% endif %} FROM project p ' "LEFT OUTER JOIN " "{% if metadata_source is defined and metadata_source is not none %}" - "`{{ metadata_source|lower }}` m ON p.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " + "{{ metadata_source|lower }} m ON p.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " "{% else %}" "`{{ business_unit|lower }}`.`sensors`.`{{ asset|lower }}_{{ data_security_level|lower }}_metadata` m ON p.`{{ tagname_column }}` = m.`{{ tagname_column }}` ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " "{% endif %}" diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/interpolate.py b/src/sdk/python/rtdip_sdk/queries/time_series/interpolate.py index 581862052..eecf42dfc 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/interpolate.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/interpolate.py @@ -38,8 +38,6 @@ def get(connection: object, parameters_dict: dict) -> pd.DataFrame: tag_names (list): List of tagname or tagnames ["tag_1", "tag_2"] start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) - sample_rate (int): (deprecated) Please use time_interval_rate instead. See below. - sample_unit (str): (deprecated) Please use time_interval_unit instead. See below. time_interval_rate (str): The time interval rate (numeric input) time_interval_unit (str): The time interval unit (second, minute, day, hour) include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False @@ -67,18 +65,6 @@ def get(connection: object, parameters_dict: dict) -> pd.DataFrame: if parameters_dict["pivot"] is True and parameters_dict["display_uom"] is True: raise ValueError("pivot True and display_uom True cannot be used together") - if "sample_rate" in parameters_dict: - logging.warning( - "Parameter sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead." - ) - parameters_dict["time_interval_rate"] = parameters_dict["sample_rate"] - - if "sample_unit" in parameters_dict: - logging.warning( - "Parameter sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead." - ) - parameters_dict["time_interval_unit"] = parameters_dict["sample_unit"] - try: query = _query_builder(parameters_dict, "interpolate") diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/plot.py b/src/sdk/python/rtdip_sdk/queries/time_series/plot.py index e3c8c5b89..13a63b6ca 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/plot.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/plot.py @@ -40,8 +40,6 @@ def get(connection: object, parameters_dict: dict) -> pd.DataFrame: tag_names (list): List of tagname or tagnames ["tag_1", "tag_2"] start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) - sample_rate (int): (deprecated) Please use time_interval_rate instead. See below. - sample_unit (str): (deprecated) Please use time_interval_unit instead. See below. time_interval_rate (str): The time interval rate (numeric input) time_interval_unit (str): The time interval unit (second, minute, day, hour) include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False @@ -67,18 +65,6 @@ def get(connection: object, parameters_dict: dict) -> pd.DataFrame: if parameters_dict["pivot"] is True and parameters_dict["display_uom"] is True: raise ValueError("pivot True and display_uom True cannot be used together") - if "sample_rate" in parameters_dict: - logging.warning( - "Parameter sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead." - ) - parameters_dict["time_interval_rate"] = parameters_dict["sample_rate"] - - if "sample_unit" in parameters_dict: - logging.warning( - "Parameter sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead." - ) - parameters_dict["time_interval_unit"] = parameters_dict["sample_unit"] - try: query = _query_builder(parameters_dict, "plot") diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/resample.py b/src/sdk/python/rtdip_sdk/queries/time_series/resample.py index 0b7f5cda7..9010e774f 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/resample.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/resample.py @@ -40,8 +40,6 @@ def get(connection: object, parameters_dict: dict) -> pd.DataFrame: tag_names (list): List of tagname or tagnames ["tag_1", "tag_2"] start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) - sample_rate (int): (deprecated) Please use time_interval_rate instead. See below. - sample_unit (str): (deprecated) Please use time_interval_unit instead. See below. time_interval_rate (str): The time interval rate (numeric input) time_interval_unit (str): The time interval unit (second, minute, day, hour) agg_method (str): Aggregation Method (first, last, avg, min, max) @@ -72,18 +70,6 @@ def get(connection: object, parameters_dict: dict) -> pd.DataFrame: if parameters_dict["pivot"] is True and parameters_dict["display_uom"] is True: raise ValueError("pivot True and display_uom True cannot be used together") - if "sample_rate" in parameters_dict: - logging.warning( - "Parameter sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead." - ) - parameters_dict["time_interval_rate"] = parameters_dict["sample_rate"] - - if "sample_unit" in parameters_dict: - logging.warning( - "Parameter sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead." - ) - parameters_dict["time_interval_unit"] = parameters_dict["sample_unit"] - try: query = _query_builder(parameters_dict, "resample") diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/summary.py b/src/sdk/python/rtdip_sdk/queries/time_series/summary.py index 95179077d..341767ff9 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/summary.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/summary.py @@ -14,15 +14,7 @@ import logging import pandas as pd - -# from ._time_series_query_builder import _query_builder - -import sys - -sys.path.insert(0, ".") -from src.sdk.python.rtdip_sdk.queries.time_series._time_series_query_builder import ( - _query_builder, -) +from ._time_series_query_builder import _query_builder def get(connection: object, parameters_dict: dict) -> pd.DataFrame: diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/time_weighted_average.py b/src/sdk/python/rtdip_sdk/queries/time_series/time_weighted_average.py index 1c49cbb50..be1083e8d 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/time_weighted_average.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/time_weighted_average.py @@ -13,7 +13,6 @@ # limitations under the License. import logging import pandas as pd - from ._time_series_query_builder import _query_builder