diff --git a/.gitignore b/.gitignore index 2afca11f..020612d9 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,7 @@ celerybeat.pid *.sage.py # Environments +test*.env .env .env.export .venv* diff --git a/backend/pdm.lock b/backend/pdm.lock index d9fb2b2a..c4cb3359 100644 --- a/backend/pdm.lock +++ b/backend/pdm.lock @@ -3453,6 +3453,20 @@ files = [ {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, ] +[[package]] +name = "pytest-dotenv" +version = "0.5.2" +summary = "A py.test plugin that parses environment files before running tests" +groups = ["test"] +dependencies = [ + "pytest>=5.0.0", + "python-dotenv>=0.9.1", +] +files = [ + {file = "pytest-dotenv-0.5.2.tar.gz", hash = "sha256:2dc6c3ac6d8764c71c6d2804e902d0ff810fa19692e95fe138aefc9b1aa73732"}, + {file = "pytest_dotenv-0.5.2-py3-none-any.whl", hash = "sha256:40a2cece120a213898afaa5407673f6bd924b1fa7eafce6bda0e8abffe2f710f"}, +] + [[package]] name = "python-crontab" version = "3.1.0" @@ -3484,7 +3498,7 @@ name = "python-dotenv" version = "1.0.0" requires_python = ">=3.8" summary = "Read key-value pairs from a .env file and set them as environment variables" -groups = ["default"] +groups = ["default", "test"] files = [ {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 6708a047..a96dbcf1 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -66,8 +66,13 @@ deploy = [ ] test = [ "pytest>=8.0.1", + "pytest-dotenv==0.5.2", ] +[tool.pytest.ini_options] +env_files = "test.env" # Load env from particular env file +addopts = "-s" + [tool.pdm.scripts] # Commands for backend backend.cmd = "./entrypoint.sh" diff --git a/backend/sample.test.env b/backend/sample.test.env new file mode 100644 index 00000000..2e6edc57 --- /dev/null +++ b/backend/sample.test.env @@ -0,0 +1,40 @@ +DB_HOST='unstract-db' +DB_USER='unstract_dev' +DB_PASSWORD='unstract_pass' +DB_NAME='unstract_db' +DB_PORT=5432 + + +REDSHIFT_HOST= +REDSHIFT_PORT="5439" +REDSHIFT_DB= +REDSHIFT_USER= +REDSHIFT_PASSWORD= + +BIGQUERY_CREDS= + +SNOWFLAKE_USER= +SNOWFLAKE_PASSWORD= +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_ROLE= +SNOWFLAKE_DB= +SNOWFLAKE_SCHEMA= +SNOWFLAKE_WAREHOUSE= + +MSSQL_SERVER= +MSSQL_PORT= +MSSQL_PASSWORD= +MSSQL_DB= +MSSQL_USER= + +MYSQL_SERVER= +MYSQL_PORT= +MYSQL_PASSWORD= +MYSQL_DB= +MYSQL_USER= + +MARIADB_SERVER= +MARIADB_PORT= +MARIADB_PASSWORD= +MARIADB_DB= +MARIADB_USER= diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.py index 84f62793..7165242e 100644 --- a/backend/workflow_manager/endpoint/constants.py +++ b/backend/workflow_manager/endpoint/constants.py @@ -5,8 +5,9 @@ class TableColumns: class DBConnectionClass: - SNOWFLAKE = "SnowflakeConnection" - BIGQUERY = "Client" + SNOWFLAKE = "SnowflakeDB" + BIGQUERY = "BigQuery" + MSSQL = "MSSQL" class Snowflake: diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.py index f852e200..28587a42 100644 --- a/backend/workflow_manager/endpoint/database_utils.py +++ b/backend/workflow_manager/endpoint/database_utils.py @@ -8,22 +8,27 @@ from utils.constants import Common from workflow_manager.endpoint.constants import ( BigQuery, DBConnectionClass, - Snowflake, TableColumns, ) -from workflow_manager.endpoint.exceptions import BigQueryTableNotFound +from workflow_manager.endpoint.db_connector_helper import DBConnectorQueryHelper +from workflow_manager.endpoint.exceptions import ( + BigQueryTableNotFound, + UnstractDBException, +) from workflow_manager.workflow.enums import AgentName, ColumnModes from unstract.connectors.databases import connectors as db_connectors +from unstract.connectors.databases.exceptions import UnstractDBConnectorException from unstract.connectors.databases.unstract_db import UnstractDB +from unstract.connectors.exceptions import ConnectorError logger = logging.getLogger(__name__) class DatabaseUtils: @staticmethod - def make_sql_values_for_query( - values: dict[str, Any], column_types: dict[str, str], cls: Any = None + def get_sql_values_for_query( + values: dict[str, Any], column_types: dict[str, str], cls_name: str ) -> dict[str, str]: """Making Sql Columns and Values for Query. @@ -51,32 +56,29 @@ class DatabaseUtils: insert into the table accordingly """ sql_values: dict[str, Any] = {} - for column in values: - if cls == DBConnectionClass.SNOWFLAKE: + if cls_name == DBConnectionClass.SNOWFLAKE: col = column.lower() type_x = column_types[col] - if type_x in Snowflake.COLUMN_TYPES: - sql_values[column] = f"'{values[column]}'" - elif type_x == "VARIANT": + if type_x == "VARIANT": values[column] = values[column].replace("'", "\\'") sql_values[column] = f"parse_json($${values[column]}$$)" else: sql_values[column] = f"{values[column]}" - elif cls == DBConnectionClass.BIGQUERY: + elif cls_name == DBConnectionClass.BIGQUERY: col = column.lower() type_x = column_types[col] if type_x in BigQuery.COLUMN_TYPES: - sql_values[column] = f"{type_x}('{values[column]}')" + sql_values[column] = f"{type_x}({values[column]})" else: - sql_values[column] = f"'{values[column]}'" + sql_values[column] = f"{values[column]}" else: # Default to Other SQL DBs # TODO: Handle numeric types with no quotes - sql_values[column] = f"'{values[column]}'" + sql_values[column] = f"{values[column]}" if column_types.get("id"): uuid_id = str(uuid.uuid4()) - sql_values["id"] = f"'{uuid_id}'" + sql_values["id"] = f"{uuid_id}" return sql_values @staticmethod @@ -96,7 +98,7 @@ class DatabaseUtils: @staticmethod def get_column_types( - cls: Any, + cls_name: Any, table_name: str, connector_id: str, connector_settings: dict[str, Any], @@ -118,7 +120,7 @@ class DatabaseUtils: """ column_types: dict[str, str] = {} try: - if cls == DBConnectionClass.SNOWFLAKE: + if cls_name == DBConnectionClass.SNOWFLAKE: query = f"describe table {table_name}" results = DatabaseUtils.execute_and_fetch_data( connector_id=connector_id, @@ -127,7 +129,7 @@ class DatabaseUtils: ) for column in results: column_types[column[0].lower()] = column[1].split("(")[0] - elif cls == DBConnectionClass.BIGQUERY: + elif cls_name == DBConnectionClass.BIGQUERY: bigquery_table_name = str.lower(table_name).split(".") if len(bigquery_table_name) != BigQuery.TABLE_NAME_SIZE: raise BigQueryTableNotFound() @@ -223,8 +225,8 @@ class DatabaseUtils: return values @staticmethod - def get_sql_columns_and_values_for_query( - engine: Any, + def get_sql_query_data( + cls_name: str, connector_id: str, connector_settings: dict[str, Any], table_name: str, @@ -234,7 +236,6 @@ class DatabaseUtils: provided values and table schema. Args: - engine (Any): The database engine. connector_id: The connector id of the connector provided connector_settings: Connector settings provided by user table_name (str): The name of the target table for the insert query. @@ -252,26 +253,26 @@ class DatabaseUtils: - For other SQL databases, it uses default SQL generation based on column types. """ - class_name = engine.__class__.__name__ column_types: dict[str, str] = DatabaseUtils.get_column_types( - cls=class_name, + cls_name=cls_name, table_name=table_name, connector_id=connector_id, connector_settings=connector_settings, ) - sql_columns_and_values = DatabaseUtils.make_sql_values_for_query( + sql_columns_and_values = DatabaseUtils.get_sql_values_for_query( values=values, column_types=column_types, - cls=class_name, + cls_name=cls_name, ) return sql_columns_and_values @staticmethod def execute_write_query( + db_class: UnstractDB, engine: Any, table_name: str, sql_keys: list[str], - sql_values: list[str], + sql_values: Any, ) -> None: """Execute Insert Query. @@ -284,22 +285,30 @@ class DatabaseUtils: - Snowflake does not support INSERT INTO ... VALUES ... syntax when VARIANT columns are present (JSON). So we need to use INSERT INTO ... SELECT ... syntax + - sql values can contain data with single quote. It needs to """ - sql = ( - f"INSERT INTO {table_name} ({','.join(sql_keys)}) " - f"SELECT {','.join(sql_values)}" + cls_name = db_class.__class__.__name__ + sql = DBConnectorQueryHelper.build_sql_insert_query( + cls_name=cls_name, table_name=table_name, sql_keys=sql_keys ) - logger.debug(f"insertng into table with: {sql} query") + logger.debug(f"inserting into table {table_name} with: {sql} query") + + sql_values = DBConnectorQueryHelper.prepare_sql_values( + cls_name=cls_name, sql_values=sql_values, sql_keys=sql_keys + ) + logger.debug(f"sql_values: {sql_values}") + try: - if hasattr(engine, "cursor"): - with engine.cursor() as cursor: - cursor.execute(sql) - engine.commit() - else: - engine.query(sql) - except Exception as e: - logger.error(f"Error while writing data: {str(e)}") - raise e + db_class.execute_query( + engine=engine, + sql_query=sql, + sql_values=sql_values, + table_name=table_name, + sql_keys=sql_keys, + ) + except UnstractDBConnectorException as e: + raise UnstractDBException(detail=e.detail) from e + logger.debug(f"sucessfully inserted into table {table_name} with: {sql} query") @staticmethod def get_db_class( @@ -315,7 +324,10 @@ class DatabaseUtils: ) -> Any: connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR] connector_class: UnstractDB = connector(connector_settings) - return connector_class.execute(query=query) + try: + return connector_class.execute(query=query) + except ConnectorError as e: + raise UnstractDBException(detail=e.message) from e @staticmethod def create_table_if_not_exists( @@ -328,7 +340,6 @@ class DatabaseUtils: Args: class_name (UnstractDB): Type of Unstract DB connector - engine (Any): _description_ table_name (str): _description_ database_entry (dict[str, Any]): _description_ @@ -338,62 +349,9 @@ class DatabaseUtils: sql = DBConnectorQueryHelper.create_table_query( conn_cls=db_class, table=table_name, database_entry=database_entry ) - logger.debug(f"creating table with: {sql} query") + logger.debug(f"creating table {table_name} with: {sql} query") try: - if hasattr(engine, "cursor"): - with engine.cursor() as cursor: - cursor.execute(sql) - engine.commit() - else: - engine.query(sql) - except Exception as e: - logger.error(f"Error while creating table: {str(e)}") - raise e - - -class DBConnectorQueryHelper: - """A class that helps to generate query for connector table operations.""" - - @staticmethod - def create_table_query( - conn_cls: UnstractDB, table: str, database_entry: dict[str, Any] - ) -> Any: - sql_query = "" - """Generate a SQL query to create a table, based on the provided - database entry. - - Args: - conn_cls (str): The database connector class. - Should be one of 'BIGQUERY', 'SNOWFLAKE', or other. - table (str): The name of the table to be created. - database_entry (dict[str, Any]): - A dictionary containing column names as keys - and their corresponding values. - - These values are used to determine the data types, - for the columns in the table. - - Returns: - str: A SQL query string to create a table with the specified name, - and column definitions. - - Note: - - Each conn_cls have it's implementation for SQL create table query - Based on the implementation, a base SQL create table query will be - created containing Permanent columns - - Each conn_cls also has a mapping to convert python datatype to - corresponding column type (string, VARCHAR etc) - - keys in database_entry will be converted to column type, and - column values will be the valus in database_entry - - base SQL create table will be appended based column type and - values, and generates a complete SQL create table query - """ - create_table_query = conn_cls.get_create_table_query(table=table) - sql_query += create_table_query - - for key, val in database_entry.items(): - if key not in TableColumns.PERMANENT_COLUMNS: - sql_type = conn_cls.sql_to_db_mapping(val) - sql_query += f"{key} {sql_type}, " - - return sql_query.rstrip(", ") + ");" + db_class.execute_query(engine=engine, sql_query=sql, sql_values=None) + except UnstractDBConnectorException as e: + raise UnstractDBException(detail=e.detail) from e + logger.debug(f"successfully created table {table_name} with: {sql} query") diff --git a/backend/workflow_manager/endpoint/db_connector_helper.py b/backend/workflow_manager/endpoint/db_connector_helper.py new file mode 100644 index 00000000..6c36a25a --- /dev/null +++ b/backend/workflow_manager/endpoint/db_connector_helper.py @@ -0,0 +1,77 @@ +from typing import Any + +from google.cloud import bigquery +from workflow_manager.endpoint.constants import DBConnectionClass, TableColumns + +from unstract.connectors.databases.unstract_db import UnstractDB + + +class DBConnectorQueryHelper: + """A class that helps to generate query for connector table operations.""" + + @staticmethod + def create_table_query( + conn_cls: UnstractDB, table: str, database_entry: dict[str, Any] + ) -> Any: + sql_query = "" + """Generate a SQL query to create a table, based on the provided + database entry. + + Args: + conn_cls (str): The database connector class. + Should be one of 'BIGQUERY', 'SNOWFLAKE', or other. + table (str): The name of the table to be created. + database_entry (dict[str, Any]): + A dictionary containing column names as keys + and their corresponding values. + + These values are used to determine the data types, + for the columns in the table. + + Returns: + str: A SQL query string to create a table with the specified name, + and column definitions. + + Note: + - Each conn_cls have it's implementation for SQL create table query + Based on the implementation, a base SQL create table query will be + created containing Permanent columns + - Each conn_cls also has a mapping to convert python datatype to + corresponding column type (string, VARCHAR etc) + - keys in database_entry will be converted to column type, and + column values will be the valus in database_entry + - base SQL create table will be appended based column type and + values, and generates a complete SQL create table query + """ + create_table_query = conn_cls.get_create_table_query(table=table) + sql_query += create_table_query + + for key, val in database_entry.items(): + if key not in TableColumns.PERMANENT_COLUMNS: + sql_type = conn_cls.sql_to_db_mapping(val) + sql_query += f"{key} {sql_type}, " + + return sql_query.rstrip(", ") + ");" + + @staticmethod + def build_sql_insert_query( + cls_name: str, table_name: str, sql_keys: list[str] + ) -> str: + keys_str = ",".join(sql_keys) + if cls_name == DBConnectionClass.BIGQUERY: + values_placeholder = ",".join(["@" + key for key in sql_keys]) + else: + values_placeholder = ",".join(["%s" for _ in sql_keys]) + return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})" + + @staticmethod + def prepare_sql_values(cls_name: str, sql_values: Any, sql_keys: list[str]) -> Any: + if cls_name == DBConnectionClass.MSSQL: + return tuple(sql_values) + elif cls_name == DBConnectionClass.BIGQUERY: + query_parameters = [ + bigquery.ScalarQueryParameter(key, "STRING", value) + for key, value in zip(sql_keys, sql_values) + ] + return bigquery.QueryJobConfig(query_parameters=query_parameters) + return sql_values diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py index 3024e7fa..df1db367 100644 --- a/backend/workflow_manager/endpoint/destination.py +++ b/backend/workflow_manager/endpoint/destination.py @@ -214,14 +214,16 @@ class DestinationConnector(BaseConnector): table_name=table_name, database_entry=values, ) - sql_columns_and_values = DatabaseUtils.get_sql_columns_and_values_for_query( - engine=engine, + cls_name = db_class.__class__.__name__ + sql_columns_and_values = DatabaseUtils.get_sql_query_data( + cls_name=cls_name, connector_id=connector_instance.connector_id, connector_settings=connector_settings, table_name=table_name, values=values, ) DatabaseUtils.execute_write_query( + db_class=db_class, engine=engine, table_name=table_name, sql_keys=list(sql_columns_and_values.keys()), diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.py index b4590503..69e332cd 100644 --- a/backend/workflow_manager/endpoint/exceptions.py +++ b/backend/workflow_manager/endpoint/exceptions.py @@ -69,3 +69,11 @@ class BigQueryTableNotFound(APIException): "Please enter correct correct bigquery table in the form " "{table}.{schema}.{database}." ) + + +class UnstractDBException(APIException): + default_detail = "Error creating/inserting to database. " + + def __init__(self, detail: str = default_detail) -> None: + status_code = 500 + super().__init__(detail=detail, code=status_code) diff --git a/backend/workflow_manager/endpoint/tests/__init__.py b/backend/workflow_manager/endpoint/tests/__init__.py new file mode 100644 index 00000000..fca2b240 --- /dev/null +++ b/backend/workflow_manager/endpoint/tests/__init__.py @@ -0,0 +1,3 @@ +from backend.celery import app as celery_app + +__all__ = ["celery_app"] diff --git a/backend/workflow_manager/endpoint/__init__.py b/backend/workflow_manager/endpoint/tests/test_database_utils/__init__.py similarity index 100% rename from backend/workflow_manager/endpoint/__init__.py rename to backend/workflow_manager/endpoint/tests/test_database_utils/__init__.py diff --git a/backend/workflow_manager/endpoint/tests/test_database_utils/base_test_db.py b/backend/workflow_manager/endpoint/tests/test_database_utils/base_test_db.py new file mode 100644 index 00000000..4040eb62 --- /dev/null +++ b/backend/workflow_manager/endpoint/tests/test_database_utils/base_test_db.py @@ -0,0 +1,160 @@ +import datetime +import json +import os +from typing import Any + +import pytest # type: ignore +from dotenv import load_dotenv + +from unstract.connectors.databases.bigquery import BigQuery +from unstract.connectors.databases.mariadb import MariaDB +from unstract.connectors.databases.mssql import MSSQL +from unstract.connectors.databases.mysql import MySQL +from unstract.connectors.databases.postgresql import PostgreSQL +from unstract.connectors.databases.redshift import Redshift +from unstract.connectors.databases.snowflake import SnowflakeDB +from unstract.connectors.databases.unstract_db import UnstractDB + +load_dotenv("test.env") + + +class BaseTestDB: + @pytest.fixture(autouse=True) + def base_setup(self) -> None: + self.postgres_creds = { + "user": os.getenv("DB_USER"), + "password": os.getenv("DB_PASSWORD"), + "host": os.getenv("DB_HOST"), + "port": os.getenv("DB_PORT"), + "database": os.getenv("DB_NAME"), + } + self.redshift_creds = { + "user": os.getenv("REDSHIFT_USER"), + "password": os.getenv("REDSHIFT_PASSWORD"), + "host": os.getenv("REDSHIFT_HOST"), + "port": os.getenv("REDSHIFT_PORT"), + "database": os.getenv("REDSHIFT_DB"), + } + self.snowflake_creds = { + "user": os.getenv("SNOWFLAKE_USER"), + "password": os.getenv("SNOWFLAKE_PASSWORD"), + "account": os.getenv("SNOWFLAKE_ACCOUNT"), + "role": os.getenv("SNOWFLAKE_ROLE"), + "database": os.getenv("SNOWFLAKE_DB"), + "schema": os.getenv("SNOWFLAKE_SCHEMA"), + "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"), + } + self.mssql_creds = { + "user": os.getenv("MSSQL_USER"), + "password": os.getenv("MSSQL_PASSWORD"), + "server": os.getenv("MSSQL_SERVER"), + "port": os.getenv("MSSQL_PORT"), + "database": os.getenv("MSSQL_DB"), + } + self.mysql_creds = { + "user": os.getenv("MYSQL_USER"), + "password": os.getenv("MYSQL_PASSWORD"), + "host": os.getenv("MYSQL_SERVER"), + "port": os.getenv("MYSQL_PORT"), + "database": os.getenv("MYSQL_DB"), + } + self.mariadb_creds = { + "user": os.getenv("MARIADB_USER"), + "password": os.getenv("MARIADB_PASSWORD"), + "host": os.getenv("MARIADB_SERVER"), + "port": os.getenv("MARIADB_PORT"), + "database": os.getenv("MARIADB_DB"), + } + self.database_entry = { + "created_by": "Unstract/DBWriter", + "created_at": datetime.datetime(2024, 5, 20, 7, 46, 57, 307998), + "data": '{"input_file": "simple.pdf", "result": "report"}', + } + valid_schema_name = "public" + invalid_schema_name = "public_1" + self.valid_postgres_creds = {**self.postgres_creds, "schema": valid_schema_name} + self.invalid_postgres_creds = { + **self.postgres_creds, + "schema": invalid_schema_name, + } + self.valid_redshift_creds = {**self.redshift_creds, "schema": valid_schema_name} + self.invalid_redshift_creds = { + **self.redshift_creds, + "schema": invalid_schema_name, + } + self.invalid_syntax_table_name = "invalid-syntax.name.test_output" + self.invalid_wrong_table_name = "database.schema.test_output" + self.valid_table_name = "test_output" + bigquery_json_str = os.getenv("BIGQUERY_CREDS", "{}") + self.bigquery_settings = json.loads(bigquery_json_str) + self.bigquery_settings["json_credentials"] = bigquery_json_str + self.valid_bigquery_table_name = "pandoras-tamer.bigquery_test.bigquery_output" + self.invalid_snowflake_db = {**self.snowflake_creds, "database": "invalid"} + self.invalid_snowflake_schema = {**self.snowflake_creds, "schema": "invalid"} + self.invalid_snowflake_warehouse = { + **self.snowflake_creds, + "warehouse": "invalid", + } + + # Gets all valid db instances except + # Bigquery (table name needs to be writted separately for bigquery) + @pytest.fixture( + params=[ + ("valid_postgres_creds", PostgreSQL), + ("snowflake_creds", SnowflakeDB), + ("mssql_creds", MSSQL), + ("mysql_creds", MySQL), + ("mariadb_creds", MariaDB), + ("valid_redshift_creds", Redshift), + ] + ) + def valid_dbs_instance(self, request: Any) -> Any: + return self.get_db_instance(request=request) + + # Gets all valid db instances except: + # Bigquery (table name needs to be writted separately for bigquery) + # Redshift (can't process more than 64KB character type) + @pytest.fixture( + params=[ + ("valid_postgres_creds", PostgreSQL), + ("snowflake_creds", SnowflakeDB), + ("mssql_creds", MSSQL), + ("mysql_creds", MySQL), + ("mariadb_creds", MariaDB), + ] + ) + def valid_dbs_instance_to_handle_large_doc(self, request: Any) -> Any: + return self.get_db_instance(request=request) + + def get_db_instance(self, request: Any) -> UnstractDB: + creds_name, db_class = request.param + creds = getattr(self, creds_name) + if not creds: + pytest.fail(f"Unknown credentials: {creds_name}") + db_instance = db_class(settings=creds) + return db_instance + + # Gets all invalid-db instances for postgres, redshift: + @pytest.fixture( + params=[ + ("invalid_postgres_creds", PostgreSQL), + ("invalid_redshift_creds", Redshift), + ] + ) + def invalid_dbs_instance(self, request: Any) -> Any: + return self.get_db_instance(request=request) + + @pytest.fixture + def valid_bigquery_db_instance(self) -> Any: + return BigQuery(settings=self.bigquery_settings) + + # Gets all invalid-db instances for snowflake: + @pytest.fixture( + params=[ + ("invalid_snowflake_db", SnowflakeDB), + ("invalid_snowflake_schema", SnowflakeDB), + ("invalid_snowflake_warehouse", SnowflakeDB), + ] + ) + def invalid_snowflake_db_instance(self, request: Any) -> Any: + return self.get_db_instance(request=request) diff --git a/backend/workflow_manager/endpoint/tests/test_database_utils/static/large_doc.txt b/backend/workflow_manager/endpoint/tests/test_database_utils/static/large_doc.txt new file mode 100644 index 00000000..3a3b67a0 --- /dev/null +++ b/backend/workflow_manager/endpoint/tests/test_database_utils/static/large_doc.txt @@ -0,0 +1 @@ +"\n\n UNITED STATES \n SECURITIES AND EXCHANGE COMMISSION \n Washington, D.C. 20549 \n\n FORM 10-Q \n\n(Mark One) \n [X] X QUARTERLY REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 \n For the quarterly period ended December 30, 2023 \n or \n [ ] TRANSITION REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 \n For the transition period from to \n Commission File Number: 001-36743 \n\n Apple Inc. \n (Exact name of Registrant as specified in its charter) \n\n California 94-2404110 \n (State or other jurisdiction (I.R.S. Employer Identification No.) \n of incorporation or organization) \n\n One Apple Park Way \n Cupertino, California 95014 \n (Address of principal executive offices) (Zip Code) \n (408) 996-1010 \n (Registrant\'s telephone number, including area code) \n\n Securities registered pursuant to Section 12(b) of the Act \n\n Title of each class symbol(s) Trading Name of each exchange on which registered \n Common Stock, $0.00001 par value per share AAPL The Nasdaq Stock Market LLC \n 0.000% Notes due 2025 The Nasdaq Stock Market LLC \n 0.875% Notes due 2025 The Nasdaq Stock Market LLC \n 1.625% Notes due 2026 The Nasdaq Stock Market LLC \n 2.000% Notes due 2027 The Nasdaq Stock Market LLC \n 1.375% Notes due 2029 The Nasdaq Stock Market LLC \n 3.050% Notes due 2029 The Nasdaq Stock Market LLC \n 0.500% Notes due 2031 The Nasdaq Stock Market LLC \n 3.600% Notes due 2042 The Nasdaq Stock Market LLC \n\nIndicate by check mark whether the Registrant (1) has filed all reports required to be filed by Section 13 or 15(d) of the Securities Exchange Act \nof 1934 during the preceding 12 months (or for such shorter period that the Registrant was required to file such reports), and (2) has been \nsubject to such filing requirements for the past 90 days. \n Yes [X] No [ ] \n\nIndicate by check mark whether the Registrant has submitted electronically every Interactive Data File required to be submitted pursuant to Rule \n405 of Regulation S-T (§232.405 of this chapter) during the preceding 12 months (or for such shorter period that the Registrant was required to \nsubmit such files). \n Yes [X] No [ ] \n<<<\n\n\nIndicate by check mark whether the Registrant is a large accelerated filer, an accelerated filer, a non-accelerated filer, a smaller reporting \ncompany, or an emerging growth company. See the definitions of "large accelerated filer," "accelerated filer," "smaller reporting company," and \n"emerging growth company" in Rule 12b-2 of the Exchange Act. \n\n Large accelerated filer [X] Accelerated filer [ ] \n Non-accelerated filer [ ] Smaller reporting company [ ] \n Emerging growth company [ ] \n\nIf an emerging growth company, indicate by check mark if the Registrant has elected not to use the extended transition period for complying with \nany new or revised financial accounting standards provided pursuant to Section 13(a) of the Exchange Act. [ ] \n\nIndicate by check mark whether the Registrant is a shell company (as defined in Rule 12b-2 of the Exchange Act). \n Yes [ ] No [X] \n\n 15,441,881,000 shares of common stock were issued and outstanding as of January 19, 2024. \n<<<\n\n\n Apple Inc. \n\n Form 10-Q \n\n For the Fiscal Quarter Ended December 30, 2023 \n TABLE OF CONTENTS \n\n Page \n Part I \nItem 1. Financial Statements 1 \nItem 2. Management\'s Discussion and Analysis of Financial Condition and Results of Operations 13 \nItem 3. Quantitative and Qualitative Disclosures About Market Risk 18 \nItem 4. Controls and Procedures 18 \n Part II \nItem 1. Legal Proceedings 19 \nItem 1A. Risk Factors 19 \nItem 2. Unregistered Sales of Equity Securities and Use of Proceeds 20 \nItem 3. Defaults Upon Senior Securities 21 \nItem 4. Mine Safety Disclosures 21 \nItem 5. Other Information 21 \nItem 6. Exhibits 21 \n<<<\n\n\nPARTI - FINANCIAL INFORMATION \n\nItem 1. Financial Statements \n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (Unaudited) \n (In millions, except number of shares, which are reflected in thousands, and per-share amounts) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\n Net sales: \n Products $ 96,458 $ 96,388 \n Services 23,117 20,766 \n Total net sales 119,575 117,154 \n\n Cost of sales: \n Products 58,440 60,765 \n Services 6,280 6,057 \n Total cost of sales 64,720 66,822 \n Gross margin 54,855 50,332 \n\n Operating expenses: \n Research and development 7,696 7,709 \n Selling, general and administrative 6,786 6,607 \n Total operating expenses 14,482 14,316 \n\n Operating income 40,373 36,016 \n Other income/(expense), net (50) (393) \n Income before provision for income taxes 40,323 35,623 \n Provision for income taxes 6,407 5,625 \n Net income $ 33,916 $ 29,998 \n\n Earnings per share: \n Basic $ 2.19 $ 1.89 \n Diluted $ 2.18 $ 1.88 \n\n Shares used in computing earnings per share: \n Basic 15,509,763 15,892,723 \n Diluted 15,576,641 15,955,718 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. IQ1 2024 Form 10-Q | 1 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (Unaudited) \n (In millions) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\nNet income $ 33,916 $ 29,998 \nOther comprehensive income/(loss): \n Change in foreign currency translation, net of tax 308 (14) \n\n Change in unrealized gains/losses on derivative instruments, net of tax: \n Change in fair value of derivative instruments (531) (988) \n Adjustment for net (gains)/losses realized and included in net income (823) (1,766) \n Total change in unrealized gains/losses on derivative instruments (1,354) (2,754) \n\n Change in unrealized gains/losses on marketable debt securities, net of tax: \n Change in fair value of marketable debt securities 3,045 900 \n Adjustment for net (gains)/losses realized and included in net income 75 65 \n Total change in unrealized gains/losses on marketable debt securities 3,120 965 \n\nTotal other comprehensive income/(loss) 2,074 (1,803) \nTotal comprehensive income $ 35,990 $ 28,195 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. I Q1 2024 Form 10-Q 12 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED BALANCE SHEETS (Unaudited) \n (In millions, except number of shares, which are reflected in thousands, and par value) \n\n December 2023 30, September 2023 30, \n\n ASSETS: \nCurrent assets: \n Cash and cash equivalents $ 40,760 $ 29,965 \n Marketable securities 32,340 31,590 \n Accounts receivable, net 23,194 29,508 \n Vendor non-trade receivables 26,908 31,477 \n Inventories 6,511 6,331 \n Other current assets 13,979 14,695 \n Total current assets 143,692 143,566 \n\nNon-current assets: \n Marketable securities 99,475 100,544 \n Property, plant and equipment, net 43,666 43,715 \n Other non-current assets 66,681 64,758 \n Total non-current assets 209,822 209,017 \n Total assets $ 353,514 $ 352,583 \n\n LIABILITIES AND SHAREHOLDERS\' EQUITY: \nCurrent liabilities: \n Accounts payable $ 58,146 $ 62,611 \n Other current liabilities 54,611 58,829 \n Deferred revenue 8,264 8,061 \n Commercial paper 1,998 5,985 \n Term debt 10,954 9,822 \n Total current liabilities 133,973 145,308 \n\nNon-current liabilities : \n Term debt 95,088 95,281 \n Other non-current liabilities 50,353 49,848 \n Total non-current liabilities 145,441 145,129 \n Total liabilities 279,414 290,437 \n\nCommitments and contingencies \n\nShareholders\' equity: \n Common stock and additional paid-in capital, $0.00001 par value: 50,400,000 shares \n authorized; 15,460,223 and 15,550,061 shares issued and outstanding, respectively 75,236 73,812 \n Retained earnings/(Accumulated deficit) 8,242 (214) \n Accumulated other comprehensive loss (9,378) (11,452) \n Total shareholders\' equity 74,100 62,146 \n Total liabilities and shareholders\' equity $ 353,514 $ 352,583 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. IQ1 2024 Form 10-Q 13 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF SHAREHOLDERS\' EQUITY (Unaudited) \n (In millions, except per-share amounts) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\nTotal shareholders\' equity, beginning balances $ 62,146 $ 50,672 \n\nCommon stock and additional paid-in capital: \n Beginning balances 73,812 64,849 \n Common stock withheld related to net share settlement of equity awards (1,660) (1,434) \n Share-based compensation 3,084 2,984 \n Ending balances 75,236 66,399 \n\nRetained earnings/(Accumulated deficit): \n Beginning balances (214) (3,068) \n Net income 33,916 29,998 \n Dividends and dividend equivalents declared (3,774) (3,712) \n Common stock withheld related to net share settlement of equity awards (1,018) (978) \n Common stock repurchased (20,668) (19,000) \n Ending balances 8,242 3,240 \n\nAccumulated other comprehensive income/(loss): \n Beginning balances (11,452) (11,109) \n Other comprehensive income/(loss) 2,074 (1,803) \n Ending balances (9,378) (12,912) \n\nTotal shareholders\' equity, ending balances $ 74,100 $ 56,727 \n\nDividends and dividend equivalents declared per share or RSU $ 0.24 $ 0.23 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. 2024 Form 10-Q 14 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF CASH FLOWS (Unaudited) \n (In millions) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\nCash, cash equivalents and restricted cash, beginning balances $ 30,737 $ 24,977 \n\nOperating activities: \n Net income 33,916 29,998 \n Adjustments to reconcile net income to cash generated by operating activities: \n Depreciation and amortization 2,848 2,916 \n Share-based compensation expense 2,997 2,905 \n Other (989) (317) \n Changes in operating assets and liabilities: \n Accounts receivable, net 6,555 4,275 \n Vendor non-trade receivables 4,569 2,320 \n Inventories (137) (1,807) \n Other current and non-current assets (1,457) (4,099) \n Accounts payable (4,542) (6,075) \n Other current and non-current liabilities (3,865) 3,889 \n Cash generated by operating activities 39,895 34,005 \n\nInvesting activities: \n Purchases of marketable securities (9,780) (5,153) \n Proceeds from maturities of marketable securities 13,046 7,127 \n Proceeds from sales of marketable securities 1,337 509 \n Payments for acquisition of property, plant and equipment (2,392) (3,787) \n Other (284) (141) \n Cash generated by/(used in) investing activities 1,927 (1,445) \n\nFinancing activities: \n Payments for taxes related to net share settlement of equity awards (2,591) (2,316) \n Payments for dividends and dividend equivalents (3,825) (3,768) \n Repurchases of common stock (20,139) (19,475) \n Repayments of term debt (1,401) \n Repayments of commercial paper, net (3,984) (8,214) \n Other (46) (389) \n Cash used in financing activities (30,585) (35,563) \n\nIncrease/(Decrease) in cash, cash equivalents and restricted cash 11,237 (3,003) \nCash, cash equivalents and restricted cash, ending balances $ 41,974 $ 21,974 \n\nSupplemental cash flow disclosure: \n Cash paid for income taxes, net $ 7,255 $ 828 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. IQ1 2024 Form 10-Q 1 5 \n<<<\n\n\n Apple Inc. \n\n Notes to Condensed Consolidated Financial Statements (Unaudited) \n\nNote 1 - Summary of Significant Accounting Policies \n\nBasis of Presentation and Preparation \nThe condensed consolidated financial statements include the accounts of Apple Inc. and its wholly owned subsidiaries \n(collectively "Apple" or the "Company"). In the opinion of the Company\'s management, the condensed consolidated financial \nstatements reflect all adjustments, which are normal and recurring in nature, necessary for fair financial statement presentation. \nThe preparation of these condensed consolidated financial statements and accompanying notes in conformity with U.S. generally \naccepted accounting principles ("GAAP") requires the use of management estimates. Certain prior period amounts in the \ncondensed consolidated financial statements and accompanying notes have been reclassified to conform to the current period\'s \npresentation. These condensed consolidated financial statements and accompanying notes should be read in conjunction with \nthe Company\'s annual consolidated financial statements and accompanying notes included in its Annual Report on Form 10-K \nfor the fiscal year ended September 30, 2023 (the "2023 Form 10-K"). \n\nThe Company\'s fiscal year is the 52- or 53-week period that ends on the last Saturday of September. An additional week is \nincluded in the first fiscal quarter every five or six years to realign the Company\'s fiscal quarters with calendar quarters, which \noccurred in the first fiscal quarter of 2023. The Company\'s fiscal years 2024 and 2023 span 52 and 53 weeks, respectively. \nUnless otherwise stated, references to particular years, quarters, months and periods refer to the Company\'s fiscal years ended \nin September and the associated quarters, months and periods of those fiscal years. \n\nNote 2 - Revenue \nNet sales disaggregated by significant products and services for the three months ended December 30, 2023 and December 31, \n2022 were as follows (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n iPhone® $ 69,702 $ 65,775 \n Mac® 7,780 7,735 \n iPad® 7,023 9,396 \n Wearables, Home and Accessories 11,953 13,482 \n Services 23,117 20,766 \n Total net sales $ 119,575 $ 117,154 \n\nTotal net sales include $3.5 billion of revenue recognized in the three months ended December 30, 2023 that was included in \ndeferred revenue as of September 30, 2023 and $3.4 billion of revenue recognized in the three months ended December 31, \n2022 that was included in deferred revenue as of September 24, 2022. \n\nThe Company\'s proportion of net sales by disaggregated revenue source was generally consistent for each reportable segment \nin Note 10, "Segment Information and Geographic Data" for the three months ended December 30, 2023 and December 31, \n2022, except in Greater China, where iPhone revenue represented a moderately higher proportion of net sales. \n\nAs of December 30, 2023 and September 30, 2023, the Company had total deferred revenue of $12.5 billion and $12.1 billion, \nrespectively. As of December 30, 2023, the Company expects 66% of total deferred revenue to be realized in less than a year, \n26% within one-to-two years, 7% within two-to-three years and 1% in greater than three years. \n\n Apple Inc. I Q1 2024 Form 10-Q | 6 \n<<<\n\n\nNote 3 - Earnings Per Share \nThe following table shows the computation of basic and diluted earnings per share for the three months ended December 30, \n2023 and December 31, 2022 (net income in millions and shares in thousands): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nNumerator: \n Net income $ 33,916 $ 29,998 \n\nDenominator: \n Weighted-average basic shares outstanding 15,509,763 15,892,723 \n Effect of dilutive share-based awards 66,878 62,995 \n Weighted-average diluted shares 15,576,641 15,955,718 \n\nBasic earnings per share $ 2.19 $ 1.89 \nDiluted earnings per share $ 2.18 $ 1.88 \n\nApproximately 89 million restricted stock units ("RSUs") were excluded from the computation of diluted earnings per share for the \nthree months ended December 31, 2022 because their effect would have been antidilutive. \n\nNote 4 - Financial Instruments \n\nCash, Cash Equivalents and Marketable Securities \nThe following tables show the Company\'s cash, cash equivalents and marketable securities by significant investment category \nas of December 30, 2023 and September 30, 2023 (in millions): \n December 30, 2023 \n Cash and Current Non-Current \n Adjusted Cost Unrealized Unrealized Fair Cash Marketable Marketable \n Gains Losses Value Equivalents Securities Securities \n Cash $ 29,542 $ $ - $ 29,542 $ 29,542 $ $ \n\nLevel 1: \n Money market funds 2,000 2,000 2,000 \n Mutual funds 448 35 (11) 472 472 \n Subtotal 2,448 35 (11) 2,472 2,000 472 \n\nLevel 2 (1): \n U.S. Treasury securities 24,041 12 (920) 23,133 7,303 4,858 10,972 \n U.S. agency securities 5,791 (448) 5,343 243 98 5,002 \n Non-U.S. government securities 17,326 54 (675) 16,705 11,175 5,530 \n Certificates of deposit and time deposits 1,448 - - 1,448 1,119 329 \n Commercial paper 1,361 1,361 472 889 \n Corporate debt securities 75,360 112 (3,964) 71,508 81 13,909 57,518 \n Municipal securities 562 (14) 548 185 363 \n Mortgage- and asset-backed securities 22,369 53 (1,907) 20,515 425 20,090 \n Subtotal 148,258 231 (7,928) 140,561 9,218 31,868 99,475 \n Total (2) $ 180,248 $ 266 $ (7,939) $ 172,575 $ 40,760 $ 32,340 $ 99,475 \n\n Apple Inc. IQ1 2024 Form 10-Q 1 7 \n<<<\n\n\n September 30, 2023 \n Cash and Current Non-Current \n Adjusted Cost Unrealized Gains Unrealized Value Fair Cash Marketable Marketable Securities \n Losses Equivalents Securities \n Cash $ 28,359 $ $ $ 28,359 $ 28,359 $ $ \n Level 1: \n Money market funds 481 481 481 \n Mutual funds and equity securities 442 12 (26) 428 428 \n Subtotal 923 12 (26) 909 481 428 \n Level 2 (1): \n U.S. Treasury securities 19,406 (1,292) 18,114 35 5,468 12,611 \n U.S. agency securities 5,736 (600) 5,136 36 271 4,829 \n Non-U.S. government securities 17,533 6 (1,048) 16,491 11,332 5,159 \n Certificates of deposit and time deposits 1,354 - 1,354 1,034 320 \n Commercial paper 608 608 608 \n Corporate debt securities 76,840 6 (5,956) 70,890 20 12,627 58,243 \n Municipal securities 628 (26) 602 192 410 \n Mortgage- and asset-backed securities 22,365 6 (2,735) 19,636 344 19,292 \n Subtotal 144,470 18 (11,657) 132,831 1,125 31,162 100,544 \n Total (2) $ 173,752 $ 30 $ (11,683) $ 162,099 $ 29,965 $ 31,590 $ 100,544 \n\n (1) The valuation techniques used to measure the fair values of the Company\'s Level 2 financial instruments, which generally \n have counterparties with high credit ratings, are based on quoted market prices or model-driven valuations using significant \n inputs derived from or corroborated by observable market data. \n (2) As of December 30, 2023 and September 30, 2023, total marketable securities included $13.9 billion and $13.8 billion, \n respectively, that were restricted from general use, related to the European Commission decision finding that Ireland granted \n state aid to the Company, and other agreements. \n\nThe following table shows the fair value of the Company\'s non-current marketable debt securities, by contractual maturity, as of \nDecember 30, 2023 (in millions): \n\n Due after 1 year through 5 years $ 72,994 \nDue after 5 years through 10 years 9,368 \nDue after 10 years 17,113 \n Total fair value $ 99,475 \n\nDerivative Instruments and Hedging \nThe Company may use derivative instruments to partially offset its business exposure to foreign exchange and interest rate risk. \nHowever, the Company may choose not to hedge certain exposures for a variety of reasons, including accounting considerations \nor the prohibitive economic cost of hedging particular exposures. There can be no assurance the hedges will offset more than a \nportion of the financial impact resulting from movements in foreign exchange or interest rates. \n\nForeign Exchange Rate Risk \nTo protect gross margins from fluctuations in foreign exchange rates, the Company may use forwards, options or other \ninstruments, and may designate these instruments as cash flow hedges. The Company generally hedges portions of its \nforecasted foreign currency exposure associated with revenue and inventory purchases, typically for up to 12 months. \n\nTo protect the Company\'s foreign currency-denominated term debt or marketable securities from fluctuations in foreign \nexchange rates, the Company may use forwards, cross-currency swaps or other instruments. The Company designates these \ninstruments as either cash flow or fair value hedges. As of December 30, 2023, the maximum length of time over which the \nCompany is hedging its exposure to the variability in future cash flows for term debt-related foreign currency transactions is 19 \nyears. \n\nThe Company may also use derivative instruments that are not designated as accounting hedges to protect gross margins from \ncertain fluctuations in foreign exchange rates, as well as to offset a portion of the foreign currency gains and losses generated by \nthe remeasurement of certain assets and liabilities denominated in non-functional currencies. \n\n Apple Inc. IQ1 2024 Form 10-Q 18 \n<<<\n\n\nInterest Rate Risk \nTo protect the Company\'s term debt or marketable securities from fluctuations in interest rates, the Company may use interest \nrate swaps, options or other instruments. The Company designates these instruments as either cash flow or fair value hedges. \n\nThe notional amounts of the Company\'s outstanding derivative instruments as of December 30, 2023 and September 30, 2023 \nwere as follows (in millions): \n December 2023 30, September 2023 30, \n\n Derivative instruments designated as accounting hedges: \n Foreign exchange contracts $ 66,735 $ 74,730 \n Interest rate contracts $ 19,375 $ 19,375 \n\n Derivative instruments not designated as accounting hedges : \n Foreign exchange contracts $ 102,108 $ 104,777 \n\n The carrying amounts of the Company\'s hedged items in fair value hedges as of December 30, 2023 and September 30, 2023 \nwere as follows (in millions): \n December 2023 30, September 2023 30, \n\n Hedged assets/(liabilities): \n Current and non-current marketable securities $ 15,102 $ 14,433 \n Current and non-current term debt $ (18,661) $ (18,247) \n\nAccounts Receivable \n\n Trade Receivables \n The Company\'s third-party cellular network carriers accounted for 34% and 41% of total trade receivables as of December 30, \n2023 and September 30, 2023, respectively. The Company requires third-party credit support or collateral from certain \ncustomers to limit credit risk. \n\n Vendor Non-Trade Receivables \nThe Company has non-trade receivables from certain of its manufacturing vendors resulting from the sale of components to \nthese vendors who manufacture subassemblies or assemble final products for the Company. The Company purchases these \ncomponents directly from suppliers. The Company does not reflect the sale of these components in products net sales. Rather, \nthe Company recognizes any gain on these sales as a reduction of products cost of sales when the related final products are \nsold by the Company. As of December 30, 2023, the Company had two vendors that individually represented 10% or more of \ntotal vendor non-trade receivables, which accounted for 50% and 20%. As of September 30, 2023, the Company had two \nvendors that individually represented 10% or more of total vendor non-trade receivables, which accounted for 48% and 23%. \n\nNote 5 - Condensed Consolidated Financial Statement Details \nThe following table shows the Company\'s condensed consolidated financial statement details as of December 30, 2023 and \nSeptember 30, 2023 (in millions): \n\nProperty, Plant and Equipment, Net \n December 2023 30, September 2023 30, \n\n Gross property, plant and equipment $ 116,176 $ 114,599 \n Accumulated depreciation (72,510) (70,884) \n Total property, plant and equipment, net $ 43,666 $ 43,715 \n\n Apple Inc. IQ1 2024 Form 10-Q 19 \n<<<\n\n\nNote 6 - Debt \n\nCommercial Paper \nThe Company issues unsecured short-term promissory notes pursuant to a commercial paper program. The Company uses net \nproceeds from the commercial paper program for general corporate purposes, including dividends and share repurchases. As of \nDecember 30, 2023 and September 30, 2023, the Company had $2.0 billion and $6.0 billion of commercial paper outstanding, \nrespectively. The following table provides a summary of cash flows associated with the issuance and maturities of commercial \npaper for the three months ended December 30, 2023 and December 31, 2022 (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Maturities 90 days or less: \n Repayments of commercial paper, net $ (3,984) $ (5,569) \n\n Maturities greater than 90 days: \n Repayments of commercial paper - (2,645) \n\n Total repayments of commercial paper, net $ (3,984) $ (8,214) \n\nTerm Debt \nAs of December 30, 2023 and September 30, 2023, the Company had outstanding fixed-rate notes with varying maturities for an \naggregate carrying amount of $106.0 billion and $105.1 billion, respectively (collectively the "Notes"). As of December 30, 2023 \nand September 30, 2023, the fair value of the Company\'s Notes, based on Level 2 inputs, was $96.7 billion and $90.8 billion, \nrespectively. \n\nNote 7 - Shareholders\' Equity \n\nShare Repurchase Program \nDuring the three months ended December 30, 2023, the Company repurchased 118 million shares of its common stock for $20.5 \nbillion. The Company\'s share repurchase program does not obligate the Company to acquire a minimum amount of shares. \nUnder the program, shares may be repurchased in privately negotiated or open market transactions, including under plans \ncomplying with Rule 10b5-1 under the Securities Exchange Act of 1934, as amended (the "Exchange Act"). \n\nNote 8 - Share-Based Compensation \n\nRestricted Stock Units \nA summary of the Company\'s RSU activity and related information for the three months ended December 30, 2023 is as follows: \n Number RSUs of Weighted-Average Grant Date Fair Aggregate Fair Value \n (in thousands) Value Per RSU (in millions) \n Balance as of September 30, 2023 180,247 $ 135.91 \n RSUs granted 74,241 $ 171.58 \n RSUs vested (42,490) $ 110.75 \n RSUs canceled (3,026) $ 109.05 \n Balance as of December 30, 2023 208,972 $ 154.09 $ 40,233 \n\nThe fair value as of the respective vesting dates of RSUs was $7.7 billion and $6.8 billion for the three months ended December \n30, 2023 and December 31, 2022, respectively. \n\n Apple Inc. I Q1 2024 Form 10-Q | 10 \n<<<\n\n\nShare-Based Compensation \nThe following table shows share-based compensation expense and the related income tax benefit included in the Condensed \nConsolidated Statements of Operations for the three months ended December 30, 2023 and December 31, 2022 (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nShare-based compensation expense $ 2,997 $ 2,905 \nIncome tax benefit related to share-based compensation expense $ (1,235) $ (1,178) \n\nAs of December 30, 2023, the total unrecognized compensation cost related to outstanding RSUs was $27.4 billion, which the \nCompany expects to recognize over a weighted-average period of 2.9 years. \n\nNote 9 - Contingencies \nThe Company is subject to various legal proceedings and claims that have arisen in the ordinary course of business and that \nhave not been fully resolved. The outcome of litigation is inherently uncertain. In the opinion of management, there was not at \nleast a reasonable possibility the Company may have incurred a material loss, or a material loss greater than a recorded accrual, \nconcerning loss contingencies for asserted legal and other claims. \n\nNote 10 - Segment Information and Geographic Data \nThe following table shows information by reportable segment for the three months ended December 30, 2023 and December 31, \n2022 (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nAmericas: \n Net sales $ 50,430 $ 49,278 \n Operating income $ 20,357 $ 17,864 \n\n Europe: \n Net sales $ 30,397 $ 27,681 \n Operating income $ 12,711 $ 10,017 \n\nGreater China: \n Net sales $ 20,819 $ 23,905 \n Operating income $ 8,622 $ 10,437 \n\nJapan: \n Net sales $ 7,767 $ 6,755 \n Operating income $ 3,819 $ 3,236 \n\nRest of Asia Pacific: \n Net sales $ 10,162 $ 9,535 \n Operating income $ 4,579 $ 3,851 \n\n Apple Inc. I Q1 2024 Form 10-Q | 11 \n<<<\n\n\nA reconciliation of the Company\'s segment operating income to the Condensed Consolidated Statements of Operations for the \nthree months ended December 30, 2023 and December 31, 2022 is as follows (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Segment operating income $ 50,088 $ 45,405 \n Research and development expense (7,696) (7,709) \n Other corporate expenses, net (2,019) (1,680) \n Total operating income $ 40,373 $ 36,016 \n\n Apple Inc. I Q1 2024 Form 10-Q | 12 \n<<<\n\n\nItem 2. Management\'s Discussion and Analysis of Financial Condition and Results of Operations \n\nThis Item and other sections of this Quarterly Report on Form 10-Q ("Form 10-Q") contain forward-looking statements, within \nthe meaning of the Private Securities Litigation Reform Act of 1995, that involve risks and uncertainties. Forward-looking \nstatements provide current expectations of future events based on certain assumptions and include any statement that does \nnot directly relate to any historical or current fact. For example, statements in this Form 10-Q regarding the potential future \nimpact of macroeconomic conditions on the Company\'s business and results of operations are forward-looking statements. \nForward-looking statements can also be identified by words such as "future," "anticipates," "believes," "estimates," "expects," \n"intends," "plans," "predicts," "will," "would," "could," "can," "may," and similar terms. Forward-looking statements are not \nguarantees of future performance and the Company\'s actual results may differ significantly from the results discussed in the \nforward-looking statements. Factors that might cause such differences include, but are not limited to, those discussed in Part I, \nItem 1A of the 2023 Form 10-K under the heading "Risk Factors." The Company assumes no obligation to revise or update any \nforward-looking statements for any reason, except as required by law. \n\nUnless otherwise stated, all information presented herein is based on the Company\'s fiscal calendar, and references to \nparticular years, quarters, months or periods refer to the Company\'s fiscal years ended in September and the associated \nquarters, months and periods of those fiscal years. \n\nThe following discussion should be read in conjunction with the 2023 Form 10-K filed with the U.S. Securities and Exchange \nCommission (the "SEC") and the condensed consolidated financial statements and accompanying notes included in Part I, \nItem 1 of this Form 10-Q. \n\nAvailable Information \nThe Company periodically provides certain information for investors on its corporate website, www.apple.com, and its investor \nrelations website, investor.apple.com. This includes press releases and other information about financial performance, \ninformation on environmental, social and governance matters, and details related to the Company\'s annual meeting of \nshareholders. The information contained on the websites referenced in this Form 10-Q is not incorporated by reference into this \nfiling. Further, the Company\'s references to website URLs are intended to be inactive textual references only. \n\nBusiness Seasonality and Product Introductions \nThe Company has historically experienced higher net sales in its first quarter compared to other quarters in its fiscal year due in \npart to seasonal holiday demand. Additionally, new product and service introductions can significantly impact net sales, cost of \nsales and operating expenses. The timing of product introductions can also impact the Company\'s net sales to its indirect \ndistribution channels as these channels are filled with new inventory following a product launch, and channel inventory of an \nolder product often declines as the launch of a newer product approaches. Net sales can also be affected when consumers and \ndistributors anticipate a product introduction. \n\nFiscal Period \nThe Company\'s fiscal year is the 52- or 53-week period that ends on the last Saturday of September. An additional week is \nincluded in the first fiscal quarter every five or six years to realign the Company\'s fiscal quarters with calendar quarters, which \noccurred in the first quarter of 2023. The Company\'s fiscal years 2024 and 2023 span 52 and 53 weeks, respectively. \n\nQuarterly Highlights \nThe Company\'s first quarter of 2024 included 13 weeks, compared to 14 weeks during the first quarter of 2023. \n\nThe Company\'s total net sales increased 2% or $2.4 billion during the first quarter of 2024 compared to the same quarter in \n2023, driven primarily by higher net sales of iPhone and Services, partially offset by lower net sales of iPad and Wearables, \nHome and Accessories. \n\nDuring the first quarter of 2024, the Company announced an updated MacBook Pro® 14-in., MacBook Pro 16-in. and iMac®. \n\nThe Company repurchased $20.5 billion of its common stock and paid dividends and dividend equivalents of $3.8 billion during \nthe first quarter of 2024. \n\nMacroeconomic Conditions \nMacroeconomic conditions, including inflation, changes in interest rates, and currency fluctuations, have directly and indirectly \nimpacted, and could in the future materially impact, the Company\'s results of operations and financial condition. \n\n Apple Inc. I Q1 2024 Form 10-Q | 13 \n<<<\n\n\nSegment Operating Performance \nThe following table shows net sales by reportable segment for the three months ended December 30, 2023 and December 31, \n2022 (dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, Change \n\n Net sales by reportable segment: \n Americas $ 50,430 $ 49,278 2 % \n Europe 30,397 27,681 10 % \n Greater China 20,819 23,905 (13)% \n Japan 7,767 6,755 15 % \n Rest of Asia Pacific 10,162 9,535 7 % \n Total net sales $ 119,575 $ 117,154 2 % \n\nAmericas \nAmericas net sales increased 2% or $1.2 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to higher net sales of Services and iPhone, partially offset by lower net sales of iPad. The strength in foreign currencies \nrelative to the U.S. dollar had a net favorable year-over-year impact on Americas net sales during the first quarter of 2024. \n\nEurope \nEurope net sales increased 10% or $2.7 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to higher net sales of iPhone. The strength in foreign currencies relative to the U.S. dollar had a net favorable year- \nover-year impact on Europe net sales during the first quarter of 2024. \n\nGreater China \nGreater China net sales decreased 13% or $3.1 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to lower net sales of iPhone, iPad and Wearables, Home and Accessories. The weakness in the renminbi relative to the \nU.S. dollar had an unfavorable year-over-year impact on Greater China net sales during the first quarter of 2024. \n\nJapan \nJapan net sales increased 15% or $1.0 billion during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto higher net sales of iPhone. The weakness in the yen relative to the U.S. dollar had an unfavorable year-over-year impact on \nJapan net sales during the first quarter of 2024. \n\nRest of Asia Pacific \nRest of Asia Pacific net sales increased 7% or $627 million during the first quarter of 2024 compared to the same quarter in 2023 \ndue primarily to higher net sales of iPhone, partially offset by lower net sales of Wearables, Home and Accessories. \n\n Apple Inc. I Q1 2024 Form 10-Q | 14 \n<<<\n\n\nProducts and Services Performance \nThe following table shows net sales by category for the three months ended December 30, 2023 and December 31, 2022 \n(dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, Change \n\nNet sales by category: \n iPhone $ 69,702 $ 65,775 6 % \n Mac 7,780 7,735 1 % \n iPad 7,023 9,396 (25)% \n Wearables, Home and Accessories 11,953 13,482 (11)% \n Services 23,117 20,766 11 % \n Total net sales $ 119,575 $ 117,154 2 % \n\niPhone \niPhone net sales increased 6% or $3.9 billion during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto higher net sales of Pro models, partially offset by lower net sales of other models. \n\nMac \nMac net sales were relatively flat during the first quarter of 2024 compared to the same quarter in 2023. \n\niPad \niPad net sales decreased 25% or $2.4 billion during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto lower net sales of iPad Pro, iPad 9th generation and iPad Air. \n\nWearables, Home and Accessories \nWearables, Home and Accessories net sales decreased 11% or $1.5 billion during the first quarter of 2024 compared to the \nsame quarter in 2023 due primarily to lower net sales of Wearables and Accessories. \n\nServices \nServices net sales increased 11% or $2.4 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to higher net sales from advertising, video and cloud services. \n\n Apple Inc. I Q1 2024 Form 10-Q | 15 \n<<<\n\n\nGross Margin \nProducts and Services gross margin and gross margin percentage for the three months ended December 30, 2023 and \nDecember 31, 2022 were as follows (dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nGross margin: \n Products $ 38,018 $ 35,623 \n Services 16,837 14,709 \n Total gross margin $ 54,855 $ 50,332 \n\n Gross margin percentage: \n Products 39.4% 37.0% \n Services 72.8% 70.8% \n Total gross margin percentage 45.9% 43.0% \n\nProducts Gross Margin \nProducts gross margin increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily to cost \nsavings and a different Products mix, partially offset by the weakness in foreign currencies relative to the U.S. dollar and lower \nProducts volume. \n\nProducts gross margin percentage increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto cost savings and a different Products mix, partially offset by the weakness in foreign currencies relative to the U.S. dollar. \n\nServices Gross Margin \nServices gross margin increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily to higher \nServices net sales and a different Services mix. \n\nServices gross margin percentage increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto a different Services mix. \n\nThe Company\'s future gross margins can be impacted by a variety of factors, as discussed in Part I, Item 1A of the 2023 Form \n10-K under the heading "Risk Factors." As a result, the Company believes, in general, gross margins will be subject to volatility \nand downward pressure. \n\n Apple Inc. I Q1 2024 Form 10-Q | 16 \n<<<\n\n\nOperating Expenses \nOperating expenses for the three months ended December 30, 2023 and December 31, 2022 were as follows (dollars in \nmillions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Research and development $ 7,696 $ 7,709 \n Percentage of total net sales 6% 7% \n\n Selling, general and administrative $ 6,786 $ 6,607 \n Percentage of total net sales 6% 6% \n Total operating expenses $ 14,482 $ 14,316 \n Percentage of total net sales 12% 12% \n\nResearch and Development \nResearch and development ("R&D") expense was relatively flat during the first quarter of 2024 compared to the same quarter in \n2023. \n\nSelling, General and Administrative \nSelling, general and administrative expense increased 3% or $179 million during the first quarter of 2024 compared to the same \nquarter in 2023. \n\nProvision for Income Taxes \nProvision for income taxes, effective tax rate and statutory federal income tax rate for the three months ended December 30, \n2023 and December 31, 2022 were as follows (dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Provision for income taxes $ 6,407 $ 5,625 \n Effective tax rate 15.9% 15.8% \n Statutory federal income tax rate 21% 21% \n\nThe Company\'s effective tax rate for the first quarter of 2024 was lower than the statutory federal income tax rate due primarily to \na lower effective tax rate on foreign earnings, tax benefits from share-based compensation, and the impact of the U.S. federal \nR&D credit, partially offset by state income taxes. \n\nThe Company\'s effective tax rate for the first quarter of 2024 was relatively flat compared to the same quarter in 2023. \n\nLiquidity and Capital Resources \nThe Company believes its balances of cash, cash equivalents and unrestricted marketable securities, along with cash generated \nby ongoing operations and continued access to debt markets, will be sufficient to satisfy its cash requirements and capital return \nprogram over the next 12 months and beyond. \n\nThe Company\'s contractual cash requirements have not changed materially since the 2023 Form 10-K, except for manufacturing \npurchase obligations. \n\nManufacturing Purchase Obligations \nThe Company utilizes several outsourcing partners to manufacture subassemblies for the Company\'s products and to perform \nfinal assembly and testing of finished products. The Company also obtains individual components for its products from a wide \nvariety of individual suppliers. As of December 30, 2023, the Company had manufacturing purchase obligations of $38.0 billion, \nwith $37.9 billion payable within 12 months. \n\n Apple Inc. I Q1 2024 Form 10-Q | 17 \n<<<\n\n\nCapital Return Program \nIn addition to its contractual cash requirements, the Company has an authorized share repurchase program. The program does \nnot obligate the Company to acquire a minimum amount of shares. As of December 30, 2023, the Company\'s quarterly cash \ndividend was $0.24 per share. The Company intends to increase its dividend on an annual basis, subject to declaration by the \nBoard of Directors. \n\nRecent Accounting Pronouncements \n\nIncome Taxes \nIn December 2023, the Financial Accounting Standards Board (the "FASB") issued Accounting Standards Update ("ASU") No. \n2023-09, Income Taxes (Topic 740): Improvements to Income Tax Disclosures ("ASU 2023-09"), which will require the Company \nto disclose specified additional information in its income tax rate reconciliation and provide additional information for reconciling \nitems that meet a quantitative threshold. ASU 2023-09 will also require the Company to disaggregate its income taxes paid \ndisclosure by federal, state and foreign taxes, with further disaggregation required for significant individual jurisdictions. The \nCompany will adopt ASU 2023-09 in its fourth quarter of 2026. ASU 2023-09 allows for adoption using either a prospective or \nretrospective transition method. \n\nSegment Reporting \nIn November 2023, the FASB issued ASU No. 2023-07, Segment Reporting (Topic 280): Improvements to Reportable Segment \nDisclosures ("ASU 2023-07\'), which will require the Company to disclose segment expenses that are significant and regularly \nprovided to the Company\'s chief operating decision maker ("CODM"). In addition, ASU 2023-07 will require the Company to \ndisclose the title and position of its CODM and how the CODM uses segment profit or loss information in assessing segment \nperformance and deciding how to allocate resources. The Company will adopt ASU 2023-07 in its fourth quarter of 2025 using a \nretrospective transition method. \n\nCritical Accounting Estimates \nThe preparation of financial statements and related disclosures in conformity with GAAP and the Company\'s discussion and \nanalysis of its financial condition and operating results require the Company\'s management to make judgments, assumptions \nand estimates that affect the amounts reported. Note 1, "Summary of Significant Accounting Policies" of the Notes to Condensed \nConsolidated Financial Statements in Part I, Item 1 of this Form 10-Q and in the Notes to Consolidated Financial Statements in \nPart II, Item 8 of the 2023 Form 10-K describe the significant accounting policies and methods used in the preparation of the \nCompany\'s condensed consolidated financial statements. There have been no material changes to the Company\'s critical \naccounting estimates since the 2023 Form 10-K. \n\nItem 3. Quantitative and Qualitative Disclosures About Market Risk \n\nThere have been no material changes to the Company\'s market risk during the first three months of 2024. For a discussion of the \nCompany\'s exposure to market risk, refer to the Company\'s market risk disclosures set forth in Part II, Item 7A, "Quantitative and \nQualitative Disclosures About Market Risk" of the 2023 Form 10-K. \n\nItem 4. Controls and Procedures \n\nEvaluation of Disclosure Controls and Procedures \nBased on an evaluation under the supervision and with the participation of the Company\'s management, the Company\'s principal \nexecutive officer and principal financial officer have concluded that the Company\'s disclosure controls and procedures as defined \nin Rules 13a-15(e) and 15d-15(e) under the Exchange Act were effective as of December 30, 2023 to provide reasonable \nassurance that information required to be disclosed by the Company in reports that it files or submits under the Exchange Act is \n(i) recorded, processed, summarized and reported within the time periods specified in the SEC rules and forms and \n(ii) accumulated and communicated to the Company\'s management, including its principal executive officer and principal \nfinancial officer, as appropriate to allow timely decisions regarding required disclosure. \n\nChanges in Internal Control over Financial Reporting \nThere were no changes in the Company\'s internal control over financial reporting during the first quarter of 2024, which were \nidentified in connection with management\'s evaluation required by paragraph (d) of Rules 13a-15 and 15d-15 under the \nExchange Act, that have materially affected, or are reasonably likely to materially affect, the Company\'s internal control over \nfinancial reporting. \n\n Apple Inc. I Q1 2024 Form 10-Q | 18 \n<<<\n\n\nPART II - OTHER INFORMATION \n\nItem 1. Legal Proceedings \n\nEpic Games \nEpic Games, Inc. ("Epic") filed a lawsuit in the U.S. District Court for the Northern District of California (the "District Court") \nagainst the Company alleging violations of federal and state antitrust laws and California\'s unfair competition law based upon the \nCompany\'s operation of its App Store®. On September 10, 2021, the District Court ruled in favor of the Company with respect to \nnine out of the ten counts included in Epic\'s claim. The District Court found that certain provisions of the Company\'s App Store \nReview Guidelines violate California\'s unfair competition law and issued an injunction enjoining the Company from prohibiting \ndevelopers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app \npurchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS® App Store. On April 24, 2023, the U.S. \nCourt of Appeals for the Ninth Circuit (the "Circuit Court") affirmed the District Court\'s ruling. On June 7, 2023, the Company and \nEpic filed petitions with the Circuit Court requesting further review of the decision. On June 30, 2023, the Circuit Court denied \nboth petitions. On July 17, 2023, the Circuit Court granted Apple\'s motion to stay enforcement of the injunction pending appeal to \nthe U.S. Supreme Court (the "Supreme Court"). On January 16, 2024, the Supreme Court denied both the Company\'s and Epic\'s \npetitions and the stay terminated. The Supreme Court\'s denial of Epic\'s petition confirms the District Court\'s ruling in favor of the \nCompany with respect to all of the antitrust claims. Following termination of the stay, the Company implemented a plan to comply \nwith the injunction and filed a statement of compliance with the District Court. On January 31, 2024, Epic filed a notice with the \nDistrict Court indicating its intent to dispute the Company\'s compliance plan. \n\nMasimo \nMasimo Corporation and Cercacor Laboratories, Inc. (together, "Masimo") filed a complaint before the U.S. International Trade \nCommission (the "ITC") alleging infringement by the Company of five patents relating to the functionality of the blood oxygen \nfeature in Apple Watch® Series 6 and 7. In its complaint, Masimo sought a permanent exclusion order prohibiting importation to \nthe U.S. of certain Apple Watch models that include blood oxygen sensing functionality. On October 26, 2023, the ITC entered a \nlimited exclusion order (the "Order") prohibiting importation and sales in the U.S. of Apple Watch models with blood oxygen \nsensing functionality, which includes Apple Watch Series 9 and Apple Watch Ultra™ 2. The Company subsequently proposed a \nredesign of Apple Watch Series 9 and Apple Watch Ultra 2 to the U.S. Customs and Border Protection (the "CBP") and appealed \nthe Order. On January 12, 2024, the CBP found that the Company\'s proposed redesign of Apple Watch Series 9 and Apple \nWatch Ultra 2 falls outside the scope of the Order, permitting the Company to import and sell the models in the U.S. \n\nOther Legal Proceedings \nThe Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the \nordinary course of business. The Company settled certain matters during the first quarter of 2024 that did not individually or in \nthe aggregate have a material impact on the Company\'s financial condition or operating results. The outcome of litigation is \ninherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above \nmanagement\'s expectations, the Company\'s financial condition and operating results for that reporting period could be materially \nadversely affected. \n\nItem 1A. Risk Factors \n\nThe Company\'s business, reputation, results of operations, financial condition and stock price can be affected by a number of \nfactors, whether currently known or unknown, including those described in Part I, Item 1A of the 2023 Form 10-K under the \nheading "Risk Factors." When any one or more of these risks materialize from time to time, the Company\'s business, reputation, \nresults of operations, financial condition and stock price can be materially and adversely affected. Except as set forth below, \nthere have been no material changes to the Company\'s risk factors since the 2023 Form 10-K. \n\nThe technology industry, including, in some instances, the Company, is subject to intense media, political and regulatory \nscrutiny, which exposes the Company to increasing regulation, government investigations, legal actions and penalties. \nFrom time to time, the Company has made changes to its App Store, including actions taken in response to litigation, \ncompetition, market conditions and legal and regulatory requirements. The Company expects to make further business changes \nin the future. For example, in the U.S. the Company has implemented changes to how developers communicate with consumers \nwithin apps on the U.S. storefront of the iOS and iPadOS App Store regarding alternative purchasing mechanisms. \n\n Apple Inc. I Q1 2024 Form 10-Q | 19 \n<<<\n\n\n In January 2024, the Company announced changes to iOS, the App Store and Safari® in the European Union to comply with the \n Digital Markets Act (the "DMA"), including new business terms and alternative fee structures for iOS apps, alternative methods of \n distribution for iOS apps, alternative payment processing for apps across the Company\'s operating systems, and additional tools \n and application programming interfaces ("APIs") for developers. Although the Company\'s compliance plan is intended to address \n the DMA\'s obligations, it is still subject to potential challenge by the European Commission or private litigants. In addition, other \n jurisdictions may seek to require the Company to make changes to its business. While the changes introduced by the Company \n in the European Union are intended to reduce new privacy and security risks the DMA poses to European Union users, many \n risks will remain. \n\n The Company is also currently subject to antitrust investigations in various jurisdictions around the world, which can result in \n legal proceedings and claims against the Company that could, individually or in the aggregate, have a materially adverse impact \n on the Company\'s business, results of operations and financial condition. For example, the Company is the subject of \n investigations in Europe and other jurisdictions relating to App Store terms and conditions. If such investigations result in adverse \nfindings against the Company, the Company could be exposed to significant fines and may be required to make further changes \n to its App Store business, all of which could materially adversely affect the Company\'s business, results of operations and \n financial condition. \n\n Further, the Company has commercial relationships with other companies in the technology industry that are or may become \n subject to investigations and litigation that, if resolved against those other companies, could materially adversely affect the \n Company\'s commercial relationships with those business partners and materially adversely affect the Company\'s business, \n results of operations and financial condition. For example, the Company earns revenue from licensing arrangements with other \n companies to offer their search services on the Company\'s platforms and applications, and certain of these arrangements are \n currently subject to government investigations and legal proceedings. \n\n There can be no assurance the Company\'s business will not be materially adversely affected, individually or in the aggregate, by \nthe outcomes of such investigations, litigation or changes to laws and regulations in the future. Changes to the Company\'s \n business practices to comply with new laws and regulations or in connection with other legal proceedings can negatively impact \n the reputation of the Company\'s products for privacy and security and otherwise adversely affect the experience for users of the \n Company\'s products and services, and result in harm to the Company\'s reputation, loss of competitive advantage, poor market \n acceptance, reduced demand for products and services, and lost sales. \n\n Item 2. Unregistered Sales of Equity Securities and Use of Proceeds \n\n Purchases of Equity Securities by the Issuer and Affiliated Purchasers \n Share repurchase activity during the three months ended December 30, 2023 was as follows (in millions, except number of \n shares, which are reflected in thousands, and per-share amounts): \n Total of Shares Number \n Purchased as Dollar Approximate Value of \n Total Number Average Price Part Announced of Publicly Yet Shares Be Purchased That May \n of Shares Paid Per Plans or Under the Plans \n Periods Purchased Share Programs or Programs (1) \n October 1, 2023 to November 4, 2023: \n August 2023 ASRs 6,498 (2) 6,498 \n Open market and privately negotiated purchases 45,970 $ 174.03 45,970 \n\n November 5, 2023 to December 2, 2023: \n Open market and privately negotiated purchases 33,797 $ 187.14 33,797 \n\n December 3, 2023 to December 30, 2023: \n Open market and privately negotiated purchases 31,782 $ 194.29 31,782 \n Total 118,047 $ 53,569 \n\n (1) As of December 30, 2023, the Company was authorized by the Board of Directors to purchase up to $90 billion of the \n Company\'s common stock under a share repurchase program announced on May 4, 2023, of which $36.4 billion had been \n utilized. The program does not obligate the Company to acquire a minimum amount of shares. Under the program, shares \n may be repurchased in privately negotiated or open market transactions, including under plans complying with Rule 10b5-1 \n under the Exchange Act. \n (2) In August 2023, the Company entered into accelerated share repurchase agreements ("ASRs") to purchase up to a total of \n $5.0 billion of the Company\'s common stock. In October 2023, the purchase periods for these ASRs ended and an additional \n 6 million shares were delivered and retired. In total, 29 million shares were delivered under these ASRs at an average \n repurchase price of $174.93 per share. \n\n Apple Inc. I Q1 2024 Form 10-Q | 20 \n<<<\n\n\nItem 3. Defaults Upon Senior Securities \n\nNone. \n\nItem 4. Mine Safety Disclosures \n\nNot applicable. \n\nItem 5. Other Information \n\nInsider Trading Arrangements \nOn November 11, 2023 and November 27, 2023, respectively, Luca Maestri, the Company\'s Senior Vice President and Chief \nFinancial Officer, and Katherine L. Adams, the Company\'s Senior Vice President and General Counsel, each entered into a \ntrading plan intended to satisfy the affirmative defense conditions of Rule 10b5-1(c) under the Exchange Act. The plans provide \nfor the sale of all shares vested during the duration of the plans pursuant to certain equity awards granted to Mr. Maestri and Ms. \nAdams, respectively, excluding any shares withheld by the Company to satisfy income tax withholding and remittance \nobligations. Mr. Maestri\'s plan will expire on December 31, 2024, and Ms. Adams\'s plan will expire on November 1, 2024, subject \nto early termination for certain specified events set forth in the plans. \n\nItem 6. Exhibits \n Incorporated by Reference \n\n Number Exhibit Form Filing Period Date/ End \n Exhibit Description Exhibit Date \n 31.1* Rule 13a-14(a) / 15d-14(a) Certification of Chief Executive Officer. \n 31.2* Rule 13a-14(a) / 15d-14(a) Certification of Chief Financial Officer. \n 32.1 ** Section 1350 Certifications of Chief Executive Officer and Chief Financial Officer. \n 101* Inline XBRL Document Set for the condensed consolidated financial statements \n and accompanying notes in Part I, Item 1, "Financial Statements" of this \n Quarterly Report on Form 10-Q. \n 104* Inline the Exhibit XBRL for 101 the Inline cover XBRL page Document of this Quarterly Set. Report on Form 10-Q, included in \n\n * Filed herewith. \n ** Furnished herewith. \n\n Apple Inc. I Q1 2024 Form 10-Q | 21 \n<<<\n\n\n SIGNATURE \n\n Pursuant to the requirements of the Securities Exchange Act of 1934, the Registrant has duly caused this report to be signed on \nits behalf by the undersigned thereunto duly authorized. \n\n Date: February 1, 2024 Apple Inc. \n\n By: /s/ Luca Maestri \n Luca Maestri \n Senior Vice President, \n Chief Financial Officer \n\n Apple Inc. I Q1 2024 Form 10-Q | 22 \n<<<\n\n\n Exhibit 31.1 \n\n CERTIFICATION \n\nI, Timothy D. Cook, certify that: \n\n1. I have reviewed this quarterly report on Form 10-Q of Apple Inc .; \n\n2. Based on my knowledge, this report does not contain any untrue statement of a material fact or omit to state a material fact \n necessary to make the statements made, in light of the circumstances under which such statements were made, not \n misleading with respect to the period covered by this report; \n\n3. Based on my knowledge, the financial statements, and other financial information included in this report, fairly present in all \n material respects the financial condition, results of operations and cash flows of the Registrant as of, and for, the periods \n presented in this report; \n\n4. The Registrant\'s other certifying officer(s) and I are responsible for establishing and maintaining disclosure controls and \n procedures (as defined in Exchange Act Rules 13a-15(e) and 15d-15(e)) and internal control over financial reporting (as \n defined in Exchange Act Rules 13a-15(f) and 15d-15(f)) for the Registrant and have: \n\n (a) Designed such disclosure controls and procedures, or caused such disclosure controls and procedures to be \n designed under our supervision, to ensure that material information relating to the Registrant, including its \n consolidated subsidiaries, is made known to us by others within those entities, particularly during the period in \n which this report is being prepared; \n\n (b) Designed such internal control over financial reporting, or caused such internal control over financial reporting \n to be designed under our supervision, to provide reasonable assurance regarding the reliability of financial \n reporting and the preparation of financial statements for external purposes in accordance with generally \n accepted accounting principles; \n\n (c) Evaluated the effectiveness of the Registrant\'s disclosure controls and procedures and presented in this report \n our conclusions about the effectiveness of the disclosure controls and procedures, as of the end of the period \n covered by this report based on such evaluation; and \n\n (d) Disclosed in this report any change in the Registrant\'s internal control over financial reporting that occurred \n during the Registrant\'s most recent fiscal quarter (the Registrant\'s fourth fiscal quarter in the case of an annual \n report) that has materially affected, or is reasonably likely to materially affect, the Registrant\'s internal control \n over financial reporting; and \n\n5. The Registrant\'s other certifying officer(s) and I have disclosed, based on our most recent evaluation of internal control over \n financial reporting, to the Registrant\'s auditors and the audit committee of the Registrant\'s board of directors (or persons \n performing the equivalent functions): \n\n (a) All significant deficiencies and material weaknesses in the design or operation of internal control over financial \n reporting which are reasonably likely to adversely affect the Registrant\'s ability to record, process, summarize \n and report financial information; and \n\n (b) Any fraud, whether or not material, that involves management or other employees who have a significant role \n in the Registrant\'s internal control over financial reporting. \n\nDate: February 1, 2024 \n\n By: /s/ Timothy D. Cook \n Timothy D. Cook \n Chief Executive Officer \n<<<\n\n\n Exhibit 31.2 \n\n CERTIFICATION \n\nI, Luca Maestri, certify that: \n\n1. I have reviewed this quarterly report on Form 10-Q of Apple Inc .; \n\n2. Based on my knowledge, this report does not contain any untrue statement of a material fact or omit to state a material fact \n necessary to make the statements made, in light of the circumstances under which such statements were made, not \n misleading with respect to the period covered by this report; \n\n3. Based on my knowledge, the financial statements, and other financial information included in this report, fairly present in all \n material respects the financial condition, results of operations and cash flows of the Registrant as of, and for, the periods \n presented in this report; \n\n4. The Registrant\'s other certifying officer(s) and I are responsible for establishing and maintaining disclosure controls and \n procedures (as defined in Exchange Act Rules 13a-15(e) and 15d-15(e)) and internal control over financial reporting (as \n defined in Exchange Act Rules 13a-15(f) and 15d-15(f)) for the Registrant and have: \n\n (a) Designed such disclosure controls and procedures, or caused such disclosure controls and procedures to be \n designed under our supervision, to ensure that material information relating to the Registrant, including its \n consolidated subsidiaries, is made known to us by others within those entities, particularly during the period in \n which this report is being prepared; \n\n (b) Designed such internal control over financial reporting, or caused such internal control over financial reporting \n to be designed under our supervision, to provide reasonable assurance regarding the reliability of financial \n reporting and the preparation of financial statements for external purposes in accordance with generally \n accepted accounting principles; \n\n (c) Evaluated the effectiveness of the Registrant\'s disclosure controls and procedures and presented in this report \n our conclusions about the effectiveness of the disclosure controls and procedures, as of the end of the period \n covered by this report based on such evaluation; and \n\n (d) Disclosed in this report any change in the Registrant\'s internal control over financial reporting that occurred \n during the Registrant\'s most recent fiscal quarter (the Registrant\'s fourth fiscal quarter in the case of an annual \n report) that has materially affected, or is reasonably likely to materially affect, the Registrant\'s internal control \n over financial reporting; and \n\n5. The Registrant\'s other certifying officer(s) and I have disclosed, based on our most recent evaluation of internal control over \n financial reporting, to the Registrant\'s auditors and the audit committee of the Registrant\'s board of directors (or persons \n performing the equivalent functions): \n\n (a) All significant deficiencies and material weaknesses in the design or operation of internal control over financial \n reporting which are reasonably likely to adversely affect the Registrant\'s ability to record, process, summarize \n and report financial information; and \n\n (b) Any fraud, whether or not material, that involves management or other employees who have a significant role \n in the Registrant\'s internal control over financial reporting. \n\nDate: February 1, 2024 \n\n By: /s/ Luca Maestri \n Luca Maestri \n Senior Vice President, \n Chief Financial Officer \n<<<\n\n\n Exhibit 32.1 \n\n CERTIFICATIONS OF CHIEF EXECUTIVE OFFICER AND CHIEF FINANCIAL OFFICER \n PURSUANT TO \n 18 U.S.C. SECTION 1350, \n AS ADOPTED PURSUANT TO \n SECTION 906 OF THE SARBANES-OXLEY ACT OF 2002 \n\nI, Timothy D. Cook, certify, as of the date hereof, pursuant to 18 U.S.C. Section 1350, as adopted pursuant to Section 906 of the \nSarbanes-Oxley Act of 2002, that the Quarterly Report of Apple Inc. on Form 10-Q for the period ended December 30, 2023 fully \ncomplies with the requirements of Section 13(a) or 15(d) of the Securities Exchange Act of 1934 and that information contained \nin such Form 10-Q fairly presents in all material respects the financial condition and results of operations of Apple Inc. at the \ndates and for the periods indicated. \n\nDate: February 1, 2024 \n\n By: /s/ Timothy D. Cook \n Timothy D. Cook \n Chief Executive Officer \n\nI, Luca Maestri, certify, as of the date hereof, pursuant to 18 U.S.C. Section 1350, as adopted pursuant to Section 906 of the \nSarbanes-Oxley Act of 2002, that the Quarterly Report of Apple Inc. on Form 10-Q for the period ended December 30, 2023 fully \ncomplies with the requirements of Section 13(a) or 15(d) of the Securities Exchange Act of 1934 and that information contained \nin such Form 10-Q fairly presents in all material respects the financial condition and results of operations of Apple Inc. at the \ndates and for the periods indicated. \n\nDate: February 1, 2024 \n\n By: /s/ Luca Maestri \n Luca Maestri \n Senior Vice President, \n Chief Financial Officer \n\nA signed original of this written statement required by Section 906 has been provided to Apple Inc. and will be retained by Apple \nInc. and furnished to the Securities and Exchange Commission or its staff upon request. \n<<<\n" diff --git a/backend/workflow_manager/endpoint/tests/test_database_utils/test_create_table_if_not_exists.py b/backend/workflow_manager/endpoint/tests/test_database_utils/test_create_table_if_not_exists.py new file mode 100644 index 00000000..90d82a35 --- /dev/null +++ b/backend/workflow_manager/endpoint/tests/test_database_utils/test_create_table_if_not_exists.py @@ -0,0 +1,97 @@ +import pytest # type: ignore +from workflow_manager.endpoint.database_utils import DatabaseUtils +from workflow_manager.endpoint.exceptions import UnstractDBException + +from unstract.connectors.databases.unstract_db import UnstractDB + +from .base_test_db import BaseTestDB + + +class TestCreateTableIfNotExists(BaseTestDB): + def test_create_table_if_not_exists_valid( + self, valid_dbs_instance: UnstractDB + ) -> None: + engine = valid_dbs_instance.get_engine() + result = DatabaseUtils.create_table_if_not_exists( + db_class=valid_dbs_instance, + engine=engine, + table_name=self.valid_table_name, + database_entry=self.database_entry, + ) + assert result is None + + def test_create_table_if_not_exists_bigquery_valid( + self, valid_bigquery_db_instance: UnstractDB + ) -> None: + engine = valid_bigquery_db_instance.get_engine() + result = DatabaseUtils.create_table_if_not_exists( + db_class=valid_bigquery_db_instance, + engine=engine, + table_name=self.valid_bigquery_table_name, + database_entry=self.database_entry, + ) + assert result is None + + def test_create_table_if_not_exists_invalid_schema( + self, invalid_dbs_instance: UnstractDB + ) -> None: + engine = invalid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.create_table_if_not_exists( + db_class=invalid_dbs_instance, + engine=engine, + table_name=self.valid_table_name, + database_entry=self.database_entry, + ) + + def test_create_table_if_not_exists_invalid_syntax( + self, valid_dbs_instance: UnstractDB + ) -> None: + engine = valid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.create_table_if_not_exists( + db_class=valid_dbs_instance, + engine=engine, + table_name=self.invalid_syntax_table_name, + database_entry=self.database_entry, + ) + + def test_create_table_if_not_exists_wrong_table_name( + self, valid_dbs_instance: UnstractDB + ) -> None: + engine = valid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.create_table_if_not_exists( + db_class=valid_dbs_instance, + engine=engine, + table_name=self.invalid_wrong_table_name, + database_entry=self.database_entry, + ) + + def test_create_table_if_not_exists_feature_not_supported( + self, invalid_dbs_instance: UnstractDB + ) -> None: + engine = invalid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.create_table_if_not_exists( + db_class=invalid_dbs_instance, + engine=engine, + table_name=self.invalid_wrong_table_name, + database_entry=self.database_entry, + ) + + def test_create_table_if_not_exists_invalid_snowflake_db( + self, invalid_snowflake_db_instance: UnstractDB + ) -> None: + engine = invalid_snowflake_db_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.create_table_if_not_exists( + db_class=invalid_snowflake_db_instance, + engine=engine, + table_name=self.invalid_wrong_table_name, + database_entry=self.database_entry, + ) + + +if __name__ == "__main__": + pytest.main() diff --git a/backend/workflow_manager/endpoint/tests/test_database_utils/test_execute_write_query.py b/backend/workflow_manager/endpoint/tests/test_database_utils/test_execute_write_query.py new file mode 100644 index 00000000..c2861581 --- /dev/null +++ b/backend/workflow_manager/endpoint/tests/test_database_utils/test_execute_write_query.py @@ -0,0 +1,169 @@ +import os +import uuid +from typing import Any + +import pytest # type: ignore +from workflow_manager.endpoint.database_utils import DatabaseUtils +from workflow_manager.endpoint.exceptions import UnstractDBException + +from unstract.connectors.databases.redshift import Redshift +from unstract.connectors.databases.unstract_db import UnstractDB + +from .base_test_db import BaseTestDB + + +class TestExecuteWriteQuery(BaseTestDB): + @pytest.fixture(autouse=True) + def setup(self, base_setup: Any) -> None: + self.sql_columns_and_values = { + "created_by": "Unstract/DBWriter", + "created_at": "2024-05-20 10:36:25.362609", + "data": '{"input_file": "simple.pdf", "result": "report"}', + "id": str(uuid.uuid4()), + } + + def test_execute_write_query_valid(self, valid_dbs_instance: Any) -> None: + engine = valid_dbs_instance.get_engine() + result = DatabaseUtils.execute_write_query( + db_class=valid_dbs_instance, + engine=engine, + table_name=self.valid_table_name, + sql_keys=list(self.sql_columns_and_values.keys()), + sql_values=list(self.sql_columns_and_values.values()), + ) + assert result is None + + def test_execute_write_query_invalid_schema( + self, invalid_dbs_instance: Any + ) -> None: + engine = invalid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.execute_write_query( + db_class=invalid_dbs_instance, + engine=engine, + table_name=self.valid_table_name, + sql_keys=list(self.sql_columns_and_values.keys()), + sql_values=list(self.sql_columns_and_values.values()), + ) + + def test_execute_write_query_invalid_syntax(self, valid_dbs_instance: Any) -> None: + engine = valid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.execute_write_query( + db_class=valid_dbs_instance, + engine=engine, + table_name=self.invalid_syntax_table_name, + sql_keys=list(self.sql_columns_and_values.keys()), + sql_values=list(self.sql_columns_and_values.values()), + ) + + def test_execute_write_query_feature_not_supported( + self, invalid_dbs_instance: Any + ) -> None: + engine = invalid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.execute_write_query( + db_class=invalid_dbs_instance, + engine=engine, + table_name=self.invalid_wrong_table_name, + sql_keys=list(self.sql_columns_and_values.keys()), + sql_values=list(self.sql_columns_and_values.values()), + ) + + def load_text_to_sql_values(self) -> dict[str, Any]: + file_path = os.path.join(os.path.dirname(__file__), "static", "large_doc.txt") + with open(file_path, encoding="utf-8") as file: + content = file.read() + sql_columns_and_values = self.sql_columns_and_values.copy() + sql_columns_and_values["data"] = content + return sql_columns_and_values + + @pytest.fixture + def valid_redshift_db_instance(self) -> Any: + return Redshift(self.valid_redshift_creds) + + def test_execute_write_query_datatype_too_large_redshift( + self, valid_redshift_db_instance: Any + ) -> None: + engine = valid_redshift_db_instance.get_engine() + sql_columns_and_values = self.load_text_to_sql_values() + with pytest.raises(UnstractDBException): + DatabaseUtils.execute_write_query( + db_class=valid_redshift_db_instance, + engine=engine, + table_name=self.valid_table_name, + sql_keys=list(sql_columns_and_values.keys()), + sql_values=list(sql_columns_and_values.values()), + ) + + def test_execute_write_query_bigquery_valid( + self, valid_bigquery_db_instance: Any + ) -> None: + engine = valid_bigquery_db_instance.get_engine() + result = DatabaseUtils.execute_write_query( + db_class=valid_bigquery_db_instance, + engine=engine, + table_name=self.valid_bigquery_table_name, + sql_keys=list(self.sql_columns_and_values.keys()), + sql_values=list(self.sql_columns_and_values.values()), + ) + assert result is None + + def test_execute_write_query_wrong_table_name( + self, valid_dbs_instance: UnstractDB + ) -> None: + engine = valid_dbs_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.execute_write_query( + db_class=valid_dbs_instance, + engine=engine, + table_name=self.invalid_wrong_table_name, + sql_keys=list(self.sql_columns_and_values.keys()), + sql_values=list(self.sql_columns_and_values.values()), + ) + + def test_execute_write_query_bigquery_large_doc( + self, valid_bigquery_db_instance: Any + ) -> None: + engine = valid_bigquery_db_instance.get_engine() + sql_columns_and_values = self.load_text_to_sql_values() + result = DatabaseUtils.execute_write_query( + db_class=valid_bigquery_db_instance, + engine=engine, + table_name=self.valid_bigquery_table_name, + sql_keys=list(sql_columns_and_values.keys()), + sql_values=list(sql_columns_and_values.values()), + ) + assert result is None + + def test_execute_write_query_invalid_snowflake_db( + self, invalid_snowflake_db_instance: UnstractDB + ) -> None: + engine = invalid_snowflake_db_instance.get_engine() + with pytest.raises(UnstractDBException): + DatabaseUtils.execute_write_query( + db_class=invalid_snowflake_db_instance, + engine=engine, + table_name=self.invalid_wrong_table_name, + sql_keys=list(self.sql_columns_and_values.keys()), + sql_values=list(self.sql_columns_and_values.values()), + ) + + # Make this function at last to cover all large doc + def test_execute_write_query_large_doc( + self, valid_dbs_instance_to_handle_large_doc: Any + ) -> None: + engine = valid_dbs_instance_to_handle_large_doc.get_engine() + sql_columns_and_values = self.load_text_to_sql_values() + result = DatabaseUtils.execute_write_query( + db_class=valid_dbs_instance_to_handle_large_doc, + engine=engine, + table_name=self.valid_table_name, + sql_keys=list(sql_columns_and_values.keys()), + sql_values=list(sql_columns_and_values.values()), + ) + assert result is None + + +if __name__ == "__main__": + pytest.main() diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py index 2cd8b75f..573fe45d 100644 --- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py +++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py @@ -1,14 +1,22 @@ import datetime import json +import logging import os from typing import Any +import google.api_core.exceptions from google.cloud import bigquery from google.cloud.bigquery import Client +from unstract.connectors.databases.exceptions import ( + BigQueryForbiddenException, + BigQueryNotFoundException, +) from unstract.connectors.databases.unstract_db import UnstractDB from unstract.connectors.exceptions import ConnectorError +logger = logging.getLogger(__name__) + class BigQuery(UnstractDB): def __init__(self, settings: dict[str, Any]): @@ -87,3 +95,24 @@ class BigQuery(UnstractDB): f"created_by string, created_at TIMESTAMP, " ) return sql_query + + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + table_name = str(kwargs.get("table_name")) + try: + if sql_values: + engine.query(sql_query, job_config=sql_values) + else: + engine.query(sql_query) + except google.api_core.exceptions.Forbidden as e: + logger.error(f"Forbidden exception in creating/inserting data: {str(e)}") + raise BigQueryForbiddenException( + detail=e.message, + table_name=table_name, + ) from e + except google.api_core.exceptions.NotFound as e: + logger.error(f"Resource not found in creating/inserting table: {str(e)}") + raise BigQueryNotFoundException( + detail=e.message, table_name=table_name + ) from e diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions.py b/unstract/connectors/src/unstract/connectors/databases/exceptions.py new file mode 100644 index 00000000..8a139377 --- /dev/null +++ b/unstract/connectors/src/unstract/connectors/databases/exceptions.py @@ -0,0 +1,92 @@ +from typing import Any + +from unstract.connectors.exceptions import ConnectorBaseException + + +class UnstractDBConnectorException(ConnectorBaseException): + """Base class for database-related exceptions from Unstract connectors.""" + + def __init__( + self, + detail: Any, + *args: Any, + **kwargs: Any, + ) -> None: + default_detail = "Error creating/inserting to database. " + user_message = default_detail if not detail else detail + super().__init__(*args, user_message=user_message, **kwargs) + self.detail = user_message + + +class InvalidSyntaxException(UnstractDBConnectorException): + + def __init__(self, detail: Any, database: Any) -> None: + default_detail = ( + f"Error creating/writing to {database}. Syntax incorrect. " + f"Please check your table-name or schema. " + ) + super().__init__(detail=default_detail + detail) + + +class InvalidSchemaException(UnstractDBConnectorException): + def __init__(self, detail: Any, database: str) -> None: + default_detail = f"Error creating/writing to {database}. Schema not valid. " + super().__init__(detail=default_detail + detail) + + +class UnderfinedTableException(UnstractDBConnectorException): + def __init__(self, detail: Any, database: str) -> None: + default_detail = ( + f"Error creating/writing to {database}. Undefined table. " + f"Please check your table-name or schema. " + ) + super().__init__(detail=default_detail + detail) + + +class ValueTooLongException(UnstractDBConnectorException): + def __init__(self, detail: Any, database: str) -> None: + default_detail = ( + f"Error creating/writing to {database}. " + f"Size of the inserted data exceeds the limit provided by the database. " + ) + super().__init__(detail=default_detail + detail) + + +class FeatureNotSupportedException(UnstractDBConnectorException): + + def __init__(self, detail: Any, database: str) -> None: + default_detail = ( + f"Error creating/writing to {database}. " + f"Feature not supported sql error. " + ) + super().__init__(detail=default_detail + detail) + + +class SnowflakeProgrammingException(UnstractDBConnectorException): + + def __init__(self, detail: Any, database: str) -> None: + default_detail = ( + f"Error creating/writing to {database}. " + f"Please check your snowflake credentials. " + ) + super().__init__(default_detail + detail) + + +class BigQueryForbiddenException(UnstractDBConnectorException): + + def __init__(self, detail: Any, table_name: str) -> None: + default_detail = ( + f"Error creating/writing to {table_name}. " + f"Access forbidden in bigquery. Please check your permissions. " + ) + super().__init__(detail=default_detail + detail) + + +class BigQueryNotFoundException(UnstractDBConnectorException): + + def __init__(self, detail: str, table_name: str) -> None: + default_detail = ( + f"Error creating/writing to {table_name}. " + f"The requested resource was not found. " + ) + super().__init__(detail=default_detail + detail) diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py b/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py new file mode 100644 index 00000000..59611f0b --- /dev/null +++ b/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py @@ -0,0 +1,20 @@ +from typing import Any + + +class ExceptionHelper: + @staticmethod + def extract_byte_exception(e: Exception) -> Any: + """_summary_ + Extract error details from byte_exception. + Used by mssql + Args: + e (Exception): _description_ + + Returns: + Any: _description_ + """ + error_message = str(e) + error_code, error_details = eval(error_message) + if isinstance(error_details, bytes): + error_details = error_details.decode("utf-8") + return error_details diff --git a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py index e926cd41..e44f4ba9 100644 --- a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py +++ b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py @@ -5,10 +5,11 @@ import pymysql from pymysql.connections import Connection from unstract.connectors.databases.mysql import MySQL +from unstract.connectors.databases.mysql_handler import MysqlHandler from unstract.connectors.databases.unstract_db import UnstractDB -class MariaDB(UnstractDB): +class MariaDB(UnstractDB, MysqlHandler): def __init__(self, settings: dict[str, Any]): super().__init__("MariaDB") @@ -74,3 +75,13 @@ class MariaDB(UnstractDB): Mysql and Mariadb share same SQL column type """ return str(MySQL.sql_to_db_mapping(value=value)) + + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + MysqlHandler.execute_query( + engine=engine, + sql_query=sql_query, + sql_values=sql_values, + database=self.database, + ) diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py index d0109b84..76740eb1 100644 --- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py +++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py @@ -1,11 +1,17 @@ +import logging import os from typing import Any import pymssql -from pymssql import Connection +import pymssql._pymssql as PyMssql +from pymssql import Connection # type: ignore +from unstract.connectors.databases.exceptions import InvalidSyntaxException +from unstract.connectors.databases.exceptions_helper import ExceptionHelper from unstract.connectors.databases.unstract_db import UnstractDB +logger = logging.getLogger(__name__) + class MSSQL(UnstractDB): def __init__(self, settings: dict[str, Any]): @@ -49,16 +55,9 @@ class MSSQL(UnstractDB): return True def get_engine(self) -> Connection: - if self.port: - return pymssql.connect( - server=self.server, - port=self.port, - user=self.user, - password=self.password, - database=self.database, - ) - return pymssql.connect( + return pymssql.connect( # type: ignore server=self.server, + port=self.port, user=self.user, password=self.password, database=self.database, @@ -74,3 +73,22 @@ class MSSQL(UnstractDB): f"created_by TEXT, created_at DATETIMEOFFSET, " ) return sql_query + + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + try: + with engine.cursor() as cursor: + if sql_values: + cursor.execute(sql_query, sql_values) + else: + cursor.execute(sql_query) + engine.commit() + except (PyMssql.ProgrammingError, PyMssql.OperationalError) as e: + error_details = ExceptionHelper.extract_byte_exception(e=e) + logger.error( + f"Invalid syntax in creating/inserting mssql data: {error_details}" + ) + raise InvalidSyntaxException( + detail=error_details, database=self.database + ) from e diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py index 72315a67..5bd08310 100644 --- a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py +++ b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py @@ -5,10 +5,11 @@ from typing import Any import pymysql from pymysql.connections import Connection +from unstract.connectors.databases.mysql_handler import MysqlHandler from unstract.connectors.databases.unstract_db import UnstractDB -class MySQL(UnstractDB): +class MySQL(UnstractDB, MysqlHandler): def __init__(self, settings: dict[str, Any]): super().__init__("MySQL") @@ -78,3 +79,13 @@ class MySQL(UnstractDB): datetime.datetime: "TIMESTAMP", } return mapping.get(python_type, "LONGTEXT") + + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + MysqlHandler.execute_query( + engine=engine, + sql_query=sql_query, + sql_values=sql_values, + database=self.database, + ) diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py b/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py new file mode 100644 index 00000000..6ec29afb --- /dev/null +++ b/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py @@ -0,0 +1,29 @@ +import logging +from typing import Any + +import pymysql.err as MysqlError + +from unstract.connectors.databases.exceptions import InvalidSyntaxException +from unstract.connectors.databases.exceptions_helper import ExceptionHelper + +logger = logging.getLogger(__name__) + + +class MysqlHandler: + @staticmethod + def execute_query( + engine: Any, sql_query: str, sql_values: Any, database: Any + ) -> None: + try: + with engine.cursor() as cursor: + if sql_values: + cursor.execute(sql_query, sql_values) + else: + cursor.execute(sql_query) + engine.commit() + except MysqlError.ProgrammingError as e: + error_details = ExceptionHelper.extract_byte_exception(e=e) + logger.error( + f"Invalid syntax in creating/inserting mysql data: {error_details}" + ) + raise InvalidSyntaxException(detail=error_details, database=database) from e diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py index 6025f5ee..d5186cfd 100644 --- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py +++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py @@ -4,10 +4,11 @@ from typing import Any import psycopg2 from psycopg2.extensions import connection +from unstract.connectors.databases.psycopg_handler import PsycoPgHandler from unstract.connectors.databases.unstract_db import UnstractDB -class PostgreSQL(UnstractDB): +class PostgreSQL(UnstractDB, PsycoPgHandler): def __init__(self, settings: dict[str, Any]): super().__init__("PostgreSQL") @@ -71,3 +72,13 @@ class PostgreSQL(UnstractDB): options=f"-c search_path={self.schema}", ) return con + + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + PsycoPgHandler.execute_query( + engine=engine, + sql_query=sql_query, + sql_values=sql_values, + database=self.database, + ) diff --git a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py new file mode 100644 index 00000000..47c911a6 --- /dev/null +++ b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py @@ -0,0 +1,50 @@ +import logging +from typing import Any + +from psycopg2 import errors as PsycopgError + +from unstract.connectors.databases.exceptions import ( + FeatureNotSupportedException, + InvalidSchemaException, + InvalidSyntaxException, + UnderfinedTableException, + ValueTooLongException, +) + +logger = logging.getLogger(__name__) + + +class PsycoPgHandler: + @staticmethod + def execute_query( + engine: Any, sql_query: str, sql_values: Any, database: Any + ) -> None: + try: + with engine.cursor() as cursor: + if sql_values: + cursor.execute(sql_query, sql_values) + else: + cursor.execute(sql_query) + engine.commit() + except PsycopgError.InvalidSchemaName as e: + logger.error(f"Invalid schema in creating table: {e.pgerror}") + raise InvalidSchemaException(detail=e.pgerror, database=database) from e + except PsycopgError.UndefinedTable as e: + logger.error(f"Undefined table in inserting: {e.pgerror}") + raise UnderfinedTableException(detail=e.pgerror, database=database) from e + except PsycopgError.SyntaxError as e: + logger.error(f"Invalid syntax in creating/inserting data: {e.pgerror}") + raise InvalidSyntaxException(detail=e.pgerror, database=database) from e + except PsycopgError.FeatureNotSupported as e: + logger.error( + f"feature not supported in creating/inserting data: {e.pgerror}" + ) + raise FeatureNotSupportedException( + detail=e.pgerror, database=database + ) from e + except ( + PsycopgError.StringDataRightTruncation, + PsycopgError.InternalError_, + ) as e: + logger.error(f"value too long for datatype: {e.pgerror}") + raise ValueTooLongException(detail=e.pgerror, database=database) from e diff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py index a56d614c..dc83ea58 100644 --- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py +++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py @@ -5,10 +5,11 @@ from typing import Any import psycopg2 from psycopg2.extensions import connection +from unstract.connectors.databases.psycopg_handler import PsycoPgHandler from unstract.connectors.databases.unstract_db import UnstractDB -class Redshift(UnstractDB): +class Redshift(UnstractDB, PsycoPgHandler): def __init__(self, settings: dict[str, Any]): super().__init__("Redshift") @@ -74,14 +75,13 @@ class Redshift(UnstractDB): str: _description_ """ python_type = type(value) - mapping = { - str: "VARCHAR(65535)", + str: "SUPER", int: "BIGINT", float: "DOUBLE PRECISION", datetime.datetime: "TIMESTAMP", } - return mapping.get(python_type, "VARCHAR(65535)") + return mapping.get(python_type, "SUPER") @staticmethod def get_create_table_query(table: str) -> str: @@ -91,3 +91,13 @@ class Redshift(UnstractDB): f"created_by VARCHAR(65535), created_at TIMESTAMP, " ) return sql_query + + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + PsycoPgHandler.execute_query( + engine=engine, + sql_query=sql_query, + sql_values=sql_values, + database=self.database, + ) diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py index ad00ea1a..a2ae457d 100644 --- a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py +++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py @@ -1,11 +1,16 @@ +import logging import os from typing import Any import snowflake.connector +import snowflake.connector.errors as SnowflakeError from snowflake.connector.connection import SnowflakeConnection +from unstract.connectors.databases.exceptions import SnowflakeProgrammingException from unstract.connectors.databases.unstract_db import UnstractDB +logger = logging.getLogger(__name__) + class SnowflakeDB(UnstractDB): def __init__(self, settings: dict[str, Any]): @@ -70,3 +75,22 @@ class SnowflakeDB(UnstractDB): f"created_by TEXT, created_at TIMESTAMP, " ) return sql_query + + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + try: + with engine.cursor() as cursor: + if sql_values: + cursor.execute(sql_query, sql_values) + else: + cursor.execute(sql_query) + engine.commit() + except SnowflakeError.ProgrammingError as e: + logger.error( + f"snowflake programming error in crearing/inserting table: " + f"{e.msg} {e.errno}" + ) + raise SnowflakeProgrammingException( + detail=e.msg, database=self.database + ) from e diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py index a9304fce..d426c26e 100644 --- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py +++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py @@ -68,7 +68,7 @@ class UnstractDB(UnstractConnector, ABC): try: self.get_engine() except Exception as e: - raise ConnectorError(str(e)) + raise ConnectorError(str(e)) from e return True def execute(self, query: str) -> Any: @@ -77,7 +77,7 @@ class UnstractDB(UnstractConnector, ABC): cursor.execute(query) return cursor.fetchall() except Exception as e: - raise ConnectorError(str(e)) + raise ConnectorError(str(e)) from e @staticmethod def sql_to_db_mapping(value: str) -> str: @@ -107,3 +107,9 @@ class UnstractDB(UnstractConnector, ABC): f"created_by TEXT, created_at TIMESTAMP, " ) return sql_query + + @abstractmethod + def execute_query( + self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any + ) -> None: + pass