diff --git a/.github/workflows/container_builds.yml b/.github/workflows/container_builds.yml index 5e161894..71787834 100644 --- a/.github/workflows/container_builds.yml +++ b/.github/workflows/container_builds.yml @@ -17,9 +17,6 @@ jobs: strategy: matrix: include: - - python_version: p39 - arch: x86 - build_platform: linux/amd64 - python_version: p310 arch: x86 build_platform: linux/amd64 diff --git a/pipeline/Serverless/02_pipeline/pipeline_build_containers.yml b/pipeline/Serverless/02_pipeline/pipeline_build_containers.yml index e8c18e31..4e21a9fd 100644 --- a/pipeline/Serverless/02_pipeline/pipeline_build_containers.yml +++ b/pipeline/Serverless/02_pipeline/pipeline_build_containers.yml @@ -67,6 +67,29 @@ build311: - dynamodb:GetItem Resource: ${self:custom.dbArn} +build312: + description: Builds the package in a container for python3.12 then uploads to S3 + image: ${ssm:/${self:service}/${self:provider.stage}/build/p312/x86/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p312/x86/digest} + timeout: 600 + memorySize: 1769 # exactly one vcpu + iamRoleStatementsName: ${self:provider.stage}-buildp312 + iamRoleStatements: + - Effect: Allow + Action: + - s3:PutObject + - s3:PutObjectAcl + Resource: ${self:custom.s3LayersArn}/* + - Effect: Allow + Action: + - s3:ListBucket + Resource: ${self:custom.s3LayersArn} + - Effect: Allow + Action: + - dynamodb:PutItem + - dynamodb:UpdateItem + - dynamodb:GetItem + Resource: ${self:custom.dbArn} + build310Arm64: description: Builds the package in a container for python3.10 then uploads to S3 @@ -99,6 +122,30 @@ build311Arm64: memorySize: 1769 # exactly one vcpu iamRoleStatementsName: ${self:provider.stage}-buildp311arm64 iamRoleStatements: + - Effect: Allow + Action: + - s3:PutObject + - s3:PutObjectAcl + Resource: ${self:custom.s3LayersArn}/* + - Effect: Allow + Action: + - s3:ListBucket + Resource: ${self:custom.s3LayersArn} + - Effect: Allow + Action: + - dynamodb:PutItem + - dynamodb:UpdateItem + - dynamodb:GetItem + Resource: ${self:custom.dbArn} + architecture: arm64 + +build312Arm64: + description: Builds the package in a container for python3.12 then uploads to S3 + image: ${ssm:/${self:service}/${self:provider.stage}/build/p312/arm64/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p312/arm64/digest} + timeout: 600 + memorySize: 1769 # exactly one vcpu + iamRoleStatementsName: ${self:provider.stage}-buildp312arm64 + iamRoleStatements: - Effect: Allow Action: - s3:PutObject diff --git a/pipeline/Serverless/state_machines/02_pipeline.yml b/pipeline/Serverless/state_machines/02_pipeline.yml index 48cf55e6..c0a66ef9 100644 --- a/pipeline/Serverless/state_machines/02_pipeline.yml +++ b/pipeline/Serverless/state_machines/02_pipeline.yml @@ -29,32 +29,33 @@ definition: ChoicePythonVersion: Type: Choice Choices: - - Variable: $.python_version - StringEquals: "p3.8" - Next: Build38 - - Variable: $.python_version - StringEquals: "p3.9" - Next: Build39 - Variable: $.python_version StringEquals: "p3.10" Next: Build310 - Variable: $.python_version StringEquals: "p3.11" Next: Build311 + - Variable: $.python_version + StringEquals: "p3.12" + Next: Build312 - Variable: $.python_version StringEquals: "p3.10-arm64" Next: Build310arm64 - Variable: $.python_version StringEquals: "p3.11-arm64" Next: Build311arm64 + - Variable: $.python_version + StringEquals: "p3.12-arm64" + Next: Build312arm64 Default: UnknownPythonVersion - Build38: + Build310: Type: Task Resource: - Fn::GetAtt: [BuildLambdaFunction, Arn] + Fn::GetAtt: [Build310LambdaFunction, Arn] Next: Deploy Retry: - ErrorEquals: + - Lambda.AWSLambdaException - States.Timeout - Lambda.ClientExecutionTimeoutException - Lambda.ServiceException @@ -67,10 +68,10 @@ definition: - ErrorEquals: - States.ALL Next: BuildFail - Build39: + Build311: Type: Task Resource: - Fn::GetAtt: [Build39LambdaFunction, Arn] + Fn::GetAtt: [Build311LambdaFunction, Arn] Next: Deploy Retry: - ErrorEquals: @@ -87,10 +88,10 @@ definition: - ErrorEquals: - States.ALL Next: BuildFail - Build310: + Build312: Type: Task Resource: - Fn::GetAtt: [Build310LambdaFunction, Arn] + Fn::GetAtt: [Build312LambdaFunction, Arn] Next: Deploy Retry: - ErrorEquals: @@ -107,14 +108,13 @@ definition: - ErrorEquals: - States.ALL Next: BuildFail - Build311: + Build310arm64: Type: Task Resource: - Fn::GetAtt: [Build311LambdaFunction, Arn] + Fn::GetAtt: [Build310Arm64LambdaFunction, Arn] Next: Deploy Retry: - ErrorEquals: - - Lambda.AWSLambdaException - States.Timeout - Lambda.ClientExecutionTimeoutException - Lambda.ServiceException @@ -127,10 +127,10 @@ definition: - ErrorEquals: - States.ALL Next: BuildFail - Build310arm64: + Build311arm64: Type: Task Resource: - Fn::GetAtt: [Build310Arm64LambdaFunction, Arn] + Fn::GetAtt: [Build311Arm64LambdaFunction, Arn] Next: Deploy Retry: - ErrorEquals: @@ -146,10 +146,10 @@ definition: - ErrorEquals: - States.ALL Next: BuildFail - Build311arm64: + Build312arm64: Type: Task Resource: - Fn::GetAtt: [Build311Arm64LambdaFunction, Arn] + Fn::GetAtt: [Build312Arm64LambdaFunction, Arn] Next: Deploy Retry: - ErrorEquals: diff --git a/pipeline/config/config.json b/pipeline/config/config.json index 49949e71..6194ddb8 100644 --- a/pipeline/config/config.json +++ b/pipeline/config/config.json @@ -1,18 +1,21 @@ { - "python_versions": ["p3.9", "p3.10", "p3.10-arm64", "p3.11", "p3.11-arm64"], - "p3.9": { - "packages_file": "packages_p39.csv" - }, + "python_versions": ["p3.10", "p3.10-arm64", "p3.11", "p3.11-arm64", "p3.12", "p3.12-arm64"], "p3.10": { "packages_file": "packages_p310.csv" }, "p3.11":{ "packages_file": "packages_p311.csv" }, + "p3.12":{ + "packages_file": "packages_p312.csv" + }, "p3.10-arm64": { "packages_file": "packages_p310-arm64.csv" }, "p3.11-arm64":{ "packages_file": "packages_p311-arm64.csv" + }, + "p3.12-arm64":{ + "packages_file": "packages_p312-arm64.csv" } } \ No newline at end of file diff --git a/pipeline/config/packages_p312-arm64.csv b/pipeline/config/packages_p312-arm64.csv new file mode 100644 index 00000000..7895dee3 --- /dev/null +++ b/pipeline/config/packages_p312-arm64.csv @@ -0,0 +1,18 @@ +Package_Name,License,Authors/Maintainers +aws-xray-sdk,Apache-2.0,AWS +aws-requests-auth,BSD,davehmuller@gmail.com +bcrypt,Apache-2.0,cryptography-dev@python.org +beautifulsoup4,MIT,Leonard Richardson +boto3,Apache-2.0,AWS +cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors +dynamodb-encryption-sdk,Apache-2.0, +idna,https://github.com/kjd/idna/blob/master/LICENSE.rst,Kim Davis kim@cynosure.com.au +jinja2,BSD,Armin Ronache ; Pallets +numpy,https://www.numpy.org/license.html,numpy-discussion@python.org +pandas,BSD, +pyqldb,Apache-2.0,AWS +redshift-connector,Apache License Version 2.0,Amazon Web Services +requests,Apache-2.0,Kenneth Reitz +mysql-connector-python,GNU GPLv2,Oracle +aliyun-python-sdk-actiontrail,Apache-2.0,Aliyun +tabulate,MIT,s.astanin@gmail.com \ No newline at end of file diff --git a/pipeline/config/packages_p312.csv b/pipeline/config/packages_p312.csv new file mode 100644 index 00000000..7895dee3 --- /dev/null +++ b/pipeline/config/packages_p312.csv @@ -0,0 +1,18 @@ +Package_Name,License,Authors/Maintainers +aws-xray-sdk,Apache-2.0,AWS +aws-requests-auth,BSD,davehmuller@gmail.com +bcrypt,Apache-2.0,cryptography-dev@python.org +beautifulsoup4,MIT,Leonard Richardson +boto3,Apache-2.0,AWS +cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors +dynamodb-encryption-sdk,Apache-2.0, +idna,https://github.com/kjd/idna/blob/master/LICENSE.rst,Kim Davis kim@cynosure.com.au +jinja2,BSD,Armin Ronache ; Pallets +numpy,https://www.numpy.org/license.html,numpy-discussion@python.org +pandas,BSD, +pyqldb,Apache-2.0,AWS +redshift-connector,Apache License Version 2.0,Amazon Web Services +requests,Apache-2.0,Kenneth Reitz +mysql-connector-python,GNU GPLv2,Oracle +aliyun-python-sdk-actiontrail,Apache-2.0,Aliyun +tabulate,MIT,s.astanin@gmail.com \ No newline at end of file diff --git a/pipeline/config/packages_p38.csv b/pipeline/config/packages_p38.csv deleted file mode 100644 index ea937991..00000000 --- a/pipeline/config/packages_p38.csv +++ /dev/null @@ -1,104 +0,0 @@ -Package_Name,License,Authors/Maintainers -aiobotocore,Apache-2.0,Nikolay Novik nickolainovik@gmail.com -aiohttp,Apache-2.0,Nikolay Kim ; Andrew Svetlov -asyncio,PSF,Guido van Rossum -attrs,MIT,Hynek Schlawack https://www.attrs.org/en/stable/ -aws-lambda-powertools,https://github.com/awslabs/aws-lambda-powertools/blob/develop/LICENSE,AWS -aws-psycopg2,MIT, -aws-xray-sdk,Apache-2.0,AWS -aws-requests-auth,BSD,davehmuller@gmail.com -arrow,Apache-2.0,Chris Smith -bcrypt,Apache-2.0,cryptography-dev@python.org -beautifulsoup4,MIT,Leonard Richardson -black,MIT,Lukasz Langa -boltons,BSD,Mahmoud Hashemi -boto3,Apache-2.0,AWS -Cerberus,ISC,Frank Sachsenheim -chevron,MIT,Noah Morrison -cloudsplaining, MIT, Kinnaird McQuade -construct,MIT,Arkadiusz Bulski; Tomer Filiba ; Corbin Simpson -crhelper,Apache-2.0,Jay McConnell -cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors -datadog,BSD,Datadog Inc -DBnomics,GNU Affero General Public License v3 (AGPLv3),DBnomics Team -dynamodb-encryption-sdk,Apache-2.0, -elasticsearch,Apache-2.0,Honza Kral ; Nick Lang -envelopes,MIT,tomekwojcik -exchangelib,BSD, Erik Cederstrand -ffmpeg-python,Apache-2.0,Karl Kroening -flashtext,MIT,Vikash Singh -geopy,MIT,Kostya Esmukov -google-auth-oauthlib,Apache-2.0,Jon Wayne -google-auth,MIT,Jon Wayne -grpcio,Apache-2.0,GRPC Authors -gspread,MIT,Anton Burnashev -idna,https://github.com/kjd/idna/blob/master/LICENSE.rst,Kim Davis kim@cynosure.com.au -itsdangerous,BSD-3-Clause,Armin Ronache ; Pallets -jellyfish,BSD, James Turk -jinja2,BSD,Armin Ronache ; Pallets -jsonschema,MIT,Julian Berman -kafka-python,Apache-2.0,Dana Powers -kazoo,Apache-2.0,Python ZK -lambda-cache,MIT,Keith Rozario -lambda-decorators,MIT,Daniel Schep -ldap3,LGPLv3, Giovanni Cannata -langdetect,Apache-2.0,Michal Mimino Danilak -loguru,MIT,Delgan -lxml,BSD, -matplotlib,PSF,matplotlib-users@python.org -mpld3,BSD, -nltk,Apache-2.0,Steven Bird -numpy,https://www.numpy.org/license.html,numpy-discussion@python.org -openpyxl,MIT,Charlie Clark -parliament,https://github.com/duo-labs/parliament/blob/master/LICENSE, -pandas,BSD, -Pillow,https://github.com/python-pillow/Pillow/blob/master/LICENSE,Alex Clark -pika,BSD,Gavin M Roy -pulp,https://github.com/coin-or/pulp/blob/master/LICENSE, -pyarrow,Apache-2.0,Apache Arrow Developers -pycountry,GNU Lesser General Public License v2 (LGPLv2) (LGPL 2.1),Christian Theune -pycryptodome,Apache-2.0, -pydantic[email],https://github.com/samuelcolvin/pydantic/blob/master/LICENSE,Samuel Colvin -PyJWT,MIT,Jose Padilla -pysftp,BSD,Jeff Hinrichs -pymongo,Apache-2.0,Bernie Hackett -PyMUPDF,AGPLv3+,Jorj Mckir -PyMySQL,MIT,Inada Naoki -PyNaCl,Apache-2.0, -pyOpenSSL,Apache-2.0,Hynek Schlawack -pyqldb,Apache-2.0,AWS -pytesseract,GPLv3,Matthias Lee -python-Levenshtein-wheels,GPLv2+,Antti Haapala -pytz,MIT,Stuart Bishop -PyYAML,MIT,Kirril Simonov -rdklib,Apache License Version 2.0,Michael Borchet -records,ISC, -redshift-connector,Apache License Version 2.0,Amazon Web Services -requests,Apache-2.0,Kenneth Reitz -requests-html,MIT, -reportlab,BSD, -shapely,BSD,Sean Gillies -simplejson,MIT,Bob Ippolito -slackclient,MIT,slack -SQLAlchemy,MIT,Mike Bayer -tablib,MIT,Kenneth Reitz -textdistance,MIT,orsinium -tinydb,MIT,Markus Siemens -tldextract,BSD,John Kurkowski -translate,MIT,Terry Yin -twilio,MIT,twilio -dropbox,MIT,Dropbox -praw,BSD,Bryce Boe -opensearch-py,Apache-2.0,Aleksei Atavin ;Rushi Agrawal ; Shephali Mittal -xarray,Apache License V2.0,Xarray -gdal,MIT,Gdal -tradologics,Apache Software License (Apache 2.0),opensource@tradologics.com -pygsheets,MIT,Nithin Murali -ccxt,MIT License (MIT),igor.kroitor@gmail.com -pybit,MIT License (MIT License),verata@pm.me -ccxtpro,MIT License (MIT),igor.kroitor@gmail.com -pdf2image,MIT License (MIT),Belval -gremlinpython,Apache-2.0,stephen mallette -PyGithub,LGPL-3.0,PyGithub -python-docx,MIT, diff --git a/pipeline/config/packages_p39.csv b/pipeline/config/packages_p39.csv deleted file mode 100644 index b4fc92bc..00000000 --- a/pipeline/config/packages_p39.csv +++ /dev/null @@ -1,51 +0,0 @@ -Package_Name,License,Authors/Maintainers -aiobotocore,Apache-2.0,Nikolay Novik nickolainovik@gmail.com -aws-xray-sdk,Apache-2.0,AWS -aws-requests-auth,BSD,davehmuller@gmail.com -bcrypt,Apache-2.0,cryptography-dev@python.org -beautifulsoup4,MIT,Leonard Richardson -boto3,Apache-2.0,AWS -cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors -DBnomics,GNU Affero General Public License v3 (AGPLv3),DBnomics Team -dynamodb-encryption-sdk,Apache-2.0, -idna,https://github.com/kjd/idna/blob/master/LICENSE.rst,Kim Davis kim@cynosure.com.au -jinja2,BSD,Armin Ronache ; Pallets -numpy,https://www.numpy.org/license.html,numpy-discussion@python.org -pandas,BSD, -pendulum,MIT, -pycountry,GNU Lesser General Public License v2 (LGPLv2) (LGPL 2.1),Christian Theune -pydantic[email],https://github.com/samuelcolvin/pydantic/blob/master/LICENSE,Samuel Colvin -pysftp,BSD,Jeff Hinrichs -pymongo,Apache-2.0,Bernie Hackett -pyqldb,Apache-2.0,AWS -PyYAML,MIT,Kirril Simonov -redshift-connector,Apache License Version 2.0,Amazon Web Services -requests,Apache-2.0,Kenneth Reitz -requests-html,MIT, -SQLAlchemy,MIT,Mike Bayer -sympy,https://github.com/sympy/sympy/blob/master/LICENSE,https://github.com/sympy/sympy/blob/master/AUTHORS -twilio,MIT,twilio -dropbox,MIT,Dropbox -praw,BSD,Bryce Boe -pygsheets,MIT,Nithin Murali -astropy,BSD,Astropy Collaboration -mysql-connector-python,GNU GPLv2,Oracle -Metaphone,BSD,Andrew Collins -usaddress,MIT,Datamade -pyephem,MIT,Brandon Craig Rhodes -gremlinpython,Apache-2.0,stephen mallette -oracledb,Apache-2.0,anthony-tuininga (Oracle) -PyJWT,MIT,Jose Padilla -kafka-python,Apache-2.0,Dana Powers -lxml,BSD, -sklearn,BSD, -transformers,Apache-2.0,Thomas Wolf -selenium,Apache-2.0,The Selenium Project -jieba,MIT,Sun Junyi -python-docx,MIT, -xmltodict,MIT,Martin Blech -dask,BSD, Matthew Rocklin -google-api-python-client,Apache-2.0,Google LLC -GeoAlchemy2,MIT,GeoAlchemy -aliyun-python-sdk-actiontrail,Apache-2.0,Aliyun -tabulate,MIT,s.astanin@gmail.com \ No newline at end of file diff --git a/pipeline/config/test_config/config.json b/pipeline/config/test_config/config.json index 9ef58b02..6194ddb8 100644 --- a/pipeline/config/test_config/config.json +++ b/pipeline/config/test_config/config.json @@ -1,21 +1,21 @@ { - "python_versions": ["p3.8","p3.9", "p3.10", "p3.10-arm64", "p3.11", "p3.11-arm64"], - "p3.8": { - "packages_file": "packages_p38.csv" - }, - "p3.9": { - "packages_file": "packages_p39.csv" - }, + "python_versions": ["p3.10", "p3.10-arm64", "p3.11", "p3.11-arm64", "p3.12", "p3.12-arm64"], "p3.10": { "packages_file": "packages_p310.csv" }, "p3.11":{ "packages_file": "packages_p311.csv" }, + "p3.12":{ + "packages_file": "packages_p312.csv" + }, "p3.10-arm64": { "packages_file": "packages_p310-arm64.csv" }, "p3.11-arm64":{ "packages_file": "packages_p311-arm64.csv" + }, + "p3.12-arm64":{ + "packages_file": "packages_p312-arm64.csv" } } \ No newline at end of file diff --git a/pipeline/config/test_config/packages_p312-arm64.csv b/pipeline/config/test_config/packages_p312-arm64.csv new file mode 100644 index 00000000..0e241654 --- /dev/null +++ b/pipeline/config/test_config/packages_p312-arm64.csv @@ -0,0 +1,3 @@ +Package_Name,License,Authors/Maintainers +boto3,Apache-2.0,AWS +cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors diff --git a/pipeline/config/test_config/packages_p312.csv b/pipeline/config/test_config/packages_p312.csv new file mode 100644 index 00000000..475db9bf --- /dev/null +++ b/pipeline/config/test_config/packages_p312.csv @@ -0,0 +1,3 @@ +Package_Name,License,Authors/Maintainers +boto3,Apache-2.0,AWS +cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors \ No newline at end of file diff --git a/pipeline/config/test_config/packages_p39.csv b/pipeline/config/test_config/packages_p39.csv deleted file mode 100644 index 5a0888b8..00000000 --- a/pipeline/config/test_config/packages_p39.csv +++ /dev/null @@ -1,3 +0,0 @@ -Package_Name,License,Authors/Maintainers -boto3,Apache-2.0,AWS -requests,Apache-2.0,Kenneth Reitz \ No newline at end of file