From 8851ac13ef84d15f07d69ab107038956f79f5e9a Mon Sep 17 00:00:00 2001 From: Jelmert Date: Thu, 8 Nov 2018 11:14:58 +0100 Subject: [PATCH 01/25] Added the awscli container for uploading and downloading backups from and to Amazon S3 --- CONTRIBUTORS.rst | 2 ++ docs/docker-postgres-backups.rst | 8 +++++ .../compose/production/aws/Dockerfile | 9 ++++++ .../production/aws/maintenance/download | 24 +++++++++++++++ .../compose/production/aws/maintenance/upload | 30 +++++++++++++++++++ {{cookiecutter.project_slug}}/production.yml | 8 +++++ 6 files changed, 81 insertions(+) create mode 100644 {{cookiecutter.project_slug}}/compose/production/aws/Dockerfile create mode 100644 {{cookiecutter.project_slug}}/compose/production/aws/maintenance/download create mode 100644 {{cookiecutter.project_slug}}/compose/production/aws/maintenance/upload diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 154fa1ac..9486b3a0 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -114,6 +114,7 @@ Listed in alphabetical order. Irfan Ahmad `@erfaan`_ @erfaan Jan Van Bruggen `@jvanbrug`_ Jens Nilsson `@phiberjenz`_ + Jelmer Draaijer `@foarsitter`_ Jerome Leclanche `@jleclanche`_ @Adys Jimmy Gitonga `@afrowave`_ @afrowave John Cass `@jcass77`_ @cass_john @@ -231,6 +232,7 @@ Listed in alphabetical order. .. _@eriol: https://github.com/eriol .. _@eyadsibai: https://github.com/eyadsibai .. _@flyudvik: https://github.com/flyudvik +.. _@foarsitter: https://github.com/foarsitter .. _@garry-cairns: https://github.com/garry-cairns .. _@garrypolley: https://github.com/garrypolley .. _@goldhand: https://github.com/goldhand diff --git a/docs/docker-postgres-backups.rst b/docs/docker-postgres-backups.rst index c1a8a5e0..246ae403 100644 --- a/docs/docker-postgres-backups.rst +++ b/docs/docker-postgres-backups.rst @@ -85,3 +85,11 @@ You will see something like :: # ... ALTER TABLE SUCCESS: The 'my_project' database has been restored from the '/backups/backup_2018_03_13T09_05_07.sql.gz' backup. + + +Backup to Amazon S3 +---------------------------------- +For uploading your backups to Amazon S3 you can use the aws cli container. There is a upload command for uploading the postgres /backups directory recursively and there is a download command for downloading a specific backup. The default S3 environment variables are used. :: + + $ docker-compose -f production.yml run --rm awscli upload + $ docker-compose -f production.yml run --rm awscli download backup_2018_03_13T09_05_07.sql.gz diff --git a/{{cookiecutter.project_slug}}/compose/production/aws/Dockerfile b/{{cookiecutter.project_slug}}/compose/production/aws/Dockerfile new file mode 100644 index 00000000..8282047b --- /dev/null +++ b/{{cookiecutter.project_slug}}/compose/production/aws/Dockerfile @@ -0,0 +1,9 @@ +FROM garland/aws-cli-docker:1.15.47 + +COPY ./compose/production/aws/maintenance /usr/local/bin/maintenance +COPY ./compose/production/postgres/maintenance/_sourced /usr/local/bin/maintenance/_sourced + +RUN chmod +x /usr/local/bin/maintenance/* + +RUN mv /usr/local/bin/maintenance/* /usr/local/bin \ + && rmdir /usr/local/bin/maintenance diff --git a/{{cookiecutter.project_slug}}/compose/production/aws/maintenance/download b/{{cookiecutter.project_slug}}/compose/production/aws/maintenance/download new file mode 100644 index 00000000..8d5ea091 --- /dev/null +++ b/{{cookiecutter.project_slug}}/compose/production/aws/maintenance/download @@ -0,0 +1,24 @@ +#!/bin/sh + +### Download a file from your Amazon S3 bucket to the postgres /backups folder +### +### Usage: +### $ docker-compose -f production.yml run --rm awscli <1> + +set -o errexit +set -o pipefail +set -o nounset + +working_dir="$(dirname ${0})" +source "${working_dir}/_sourced/constants.sh" +source "${working_dir}/_sourced/messages.sh" + +export AWS_ACCESS_KEY_ID="${DJANGO_AWS_ACCESS_KEY_ID}" +export AWS_SECRET_ACCESS_KEY="${DJANGO_AWS_SECRET_ACCESS_KEY}" +export AWS_STORAGE_BUCKET_NAME="${DJANGO_AWS_STORAGE_BUCKET_NAME}" + + +aws s3 cp s3://${AWS_STORAGE_BUCKET_NAME}${BACKUP_DIR_PATH}/${1} ${BACKUP_DIR_PATH}/${1} + +message_success "Finished downloading ${1}." + diff --git a/{{cookiecutter.project_slug}}/compose/production/aws/maintenance/upload b/{{cookiecutter.project_slug}}/compose/production/aws/maintenance/upload new file mode 100644 index 00000000..4a89dcb5 --- /dev/null +++ b/{{cookiecutter.project_slug}}/compose/production/aws/maintenance/upload @@ -0,0 +1,30 @@ +#!/bin/sh + +### Upload the /backups folder to Amazon S3 +### +### Usage: +### $ docker-compose -f production.yml run --rm awscli upload + +set -o errexit +set -o pipefail +set -o nounset + +working_dir="$(dirname ${0})" +source "${working_dir}/_sourced/constants.sh" +source "${working_dir}/_sourced/messages.sh" + +export AWS_ACCESS_KEY_ID="${DJANGO_AWS_ACCESS_KEY_ID}" +export AWS_SECRET_ACCESS_KEY="${DJANGO_AWS_SECRET_ACCESS_KEY}" +export AWS_STORAGE_BUCKET_NAME="${DJANGO_AWS_STORAGE_BUCKET_NAME}" + + +message_info "Upload the backups directory to S3 bucket {$AWS_STORAGE_BUCKET_NAME}" + +aws s3 cp ${BACKUP_DIR_PATH} s3://${AWS_STORAGE_BUCKET_NAME}${BACKUP_DIR_PATH} --recursive + +message_info "Cleaning the directory ${BACKUP_DIR_PATH}" + +rm -rf ${BACKUP_DIR_PATH}/* + +message_success "Finished uploading and cleaning." + diff --git a/{{cookiecutter.project_slug}}/production.yml b/{{cookiecutter.project_slug}}/production.yml index 88cf8773..a24ba829 100644 --- a/{{cookiecutter.project_slug}}/production.yml +++ b/{{cookiecutter.project_slug}}/production.yml @@ -65,3 +65,11 @@ services: command: /start-flower {%- endif %} + awscli: + build: + context: . + dockerfile: ./compose/production/aws/Dockerfile + env_file: + - ./.envs/.production/.django + volumes: + - production_postgres_data_backups:/backups From 27d806fab11734440cb97e987ea2886182fc3811 Mon Sep 17 00:00:00 2001 From: Jelmert Date: Fri, 23 Nov 2018 11:19:54 +0100 Subject: [PATCH 02/25] Typo --- docs/docker-postgres-backups.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker-postgres-backups.rst b/docs/docker-postgres-backups.rst index 246ae403..6ccb7cf1 100644 --- a/docs/docker-postgres-backups.rst +++ b/docs/docker-postgres-backups.rst @@ -89,7 +89,7 @@ You will see something like :: Backup to Amazon S3 ---------------------------------- -For uploading your backups to Amazon S3 you can use the aws cli container. There is a upload command for uploading the postgres /backups directory recursively and there is a download command for downloading a specific backup. The default S3 environment variables are used. :: +For uploading your backups to Amazon S3 you can use the aws cli container. There is an upload command for uploading the postgres /backups directory recursively and there is a download command for downloading a specific backup. The default S3 environment variables are used. :: $ docker-compose -f production.yml run --rm awscli upload $ docker-compose -f production.yml run --rm awscli download backup_2018_03_13T09_05_07.sql.gz From e7115fcee93b6d3d0f82a97bb9d97ab2fe28efc9 Mon Sep 17 00:00:00 2001 From: browniebroke Date: Wed, 3 Apr 2019 12:00:30 +0100 Subject: [PATCH 03/25] Update werkzeug from 0.15.1 to 0.15.2 --- {{cookiecutter.project_slug}}/requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index 028886b0..01bb6d2f 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -1,6 +1,6 @@ -r ./base.txt -Werkzeug==0.15.1 # https://github.com/pallets/werkzeug +Werkzeug==0.15.2 # https://github.com/pallets/werkzeug ipdb==0.12 # https://github.com/gotcha/ipdb Sphinx==2.0.0 # https://github.com/sphinx-doc/sphinx {%- if cookiecutter.use_docker == 'y' %} From 0aafad16dd5f7d93f02dee9cb268dee9fa8ce2b0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 3 Apr 2019 10:24:10 -0700 Subject: [PATCH 04/25] Update tox from 3.8.4 to 3.8.6 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aebe0e54..c38f0650 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ flake8==3.7.6 # Testing # ------------------------------------------------------------------------------ -tox==3.8.4 +tox==3.8.6 pytest==4.4.0 pytest_cases==1.5.1 pytest-cookies==0.3.0 From c80bcb8932856c405cb01862340a18ffe2a90d06 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 3 Apr 2019 10:24:15 -0700 Subject: [PATCH 05/25] Update pytest_cases from 1.5.1 to 1.6.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aebe0e54..efeb48f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,6 @@ flake8==3.7.6 # ------------------------------------------------------------------------------ tox==3.8.4 pytest==4.4.0 -pytest_cases==1.5.1 +pytest_cases==1.6.1 pytest-cookies==0.3.0 pyyaml==5.1 From ea749071c2b681b5b7a576eb62279d8c8c53f6dd Mon Sep 17 00:00:00 2001 From: browniebroke Date: Thu, 4 Apr 2019 12:00:30 +0100 Subject: [PATCH 06/25] Update psycopg2-binary from 2.7.7 to 2.8 --- {{cookiecutter.project_slug}}/requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index 01bb6d2f..05ddb714 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -6,7 +6,7 @@ Sphinx==2.0.0 # https://github.com/sphinx-doc/sphinx {%- if cookiecutter.use_docker == 'y' %} psycopg2==2.7.4 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 {%- else %} -psycopg2-binary==2.7.7 # https://github.com/psycopg/psycopg2 +psycopg2-binary==2.8 # https://github.com/psycopg/psycopg2 {%- endif %} # Testing From 63f5ad028039da2ad52b659046e87f66de571686 Mon Sep 17 00:00:00 2001 From: browniebroke Date: Thu, 4 Apr 2019 12:00:33 +0100 Subject: [PATCH 07/25] Update mypy from 0.670 to 0.700 --- {{cookiecutter.project_slug}}/requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index 01bb6d2f..7e1cc25a 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -11,7 +11,7 @@ psycopg2-binary==2.7.7 # https://github.com/psycopg/psycopg2 # Testing # ------------------------------------------------------------------------------ -mypy==0.670 # https://github.com/python/mypy +mypy==0.700 # https://github.com/python/mypy pytest==4.4.0 # https://github.com/pytest-dev/pytest pytest-sugar==0.9.2 # https://github.com/Frozenball/pytest-sugar From b9c5ca2b86dd5c347d8317fe59597a4a9d59aa15 Mon Sep 17 00:00:00 2001 From: Bruno Alla Date: Thu, 4 Apr 2019 15:03:49 +0200 Subject: [PATCH 08/25] Also update psycopg2 no-binary --- {{cookiecutter.project_slug}}/requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index 05ddb714..2d7e3ba1 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -4,7 +4,7 @@ Werkzeug==0.15.2 # https://github.com/pallets/werkzeug ipdb==0.12 # https://github.com/gotcha/ipdb Sphinx==2.0.0 # https://github.com/sphinx-doc/sphinx {%- if cookiecutter.use_docker == 'y' %} -psycopg2==2.7.4 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 +psycopg2==2.8 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 {%- else %} psycopg2-binary==2.8 # https://github.com/psycopg/psycopg2 {%- endif %} From b30b9795d2f142bd06bf8e431105cf97f68163bd Mon Sep 17 00:00:00 2001 From: Bruno Alla Date: Thu, 4 Apr 2019 15:04:57 +0200 Subject: [PATCH 09/25] Update psycopg2 to 2.8 in production --- {{cookiecutter.project_slug}}/requirements/production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/production.txt b/{{cookiecutter.project_slug}}/requirements/production.txt index 035cefd7..5a40f07a 100644 --- a/{{cookiecutter.project_slug}}/requirements/production.txt +++ b/{{cookiecutter.project_slug}}/requirements/production.txt @@ -3,7 +3,7 @@ -r ./base.txt gunicorn==19.9.0 # https://github.com/benoitc/gunicorn -psycopg2==2.7.4 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 +psycopg2==2.8 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 {%- if cookiecutter.use_whitenoise == 'n' %} Collectfast==0.6.2 # https://github.com/antonagestam/collectfast {%- endif %} From cad0b7765878fdb2f8b19230d6ce9a52f3143c07 Mon Sep 17 00:00:00 2001 From: anuj Date: Tue, 2 Apr 2019 11:43:31 +0530 Subject: [PATCH 10/25] Added support for GCE Media storage --- cookiecutter.json | 4 ++++ .../.envs/.production/.django | 9 ++++++-- .../config/settings/production.py | 23 ++++++++++++------- .../requirements/production.txt | 4 ++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index b5dda0c7..eabf7d14 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -32,6 +32,10 @@ "None", "Gulp" ], + "cloud_provider": [ + "AWS", + "GCE" + ], "custom_bootstrap_compilation": "n", "use_compressor": "n", "use_celery": "n", diff --git a/{{cookiecutter.project_slug}}/.envs/.production/.django b/{{cookiecutter.project_slug}}/.envs/.production/.django index 4175f894..a938ada6 100644 --- a/{{cookiecutter.project_slug}}/.envs/.production/.django +++ b/{{cookiecutter.project_slug}}/.envs/.production/.django @@ -16,13 +16,18 @@ DJANGO_SECURE_SSL_REDIRECT=False MAILGUN_API_KEY= DJANGO_SERVER_EMAIL= MAILGUN_DOMAIN= - +{% if cookiecutter.cloud_provider == 'AWS' %} # AWS # ------------------------------------------------------------------------------ DJANGO_AWS_ACCESS_KEY_ID= DJANGO_AWS_SECRET_ACCESS_KEY= DJANGO_AWS_STORAGE_BUCKET_NAME= - +{% elif cookiecutter.cloud_provider == 'GCE' %} +# GCE +# ------------------------------------------------------------------------------ +GOOGLE_APPLICATION_CREDENTIALS= +DJANGO_GCE_STORAGE_BUCKET_NAME= +{% endif %} # django-allauth # ------------------------------------------------------------------------------ DJANGO_ACCOUNT_ALLOW_REGISTRATION=True diff --git a/{{cookiecutter.project_slug}}/config/settings/production.py b/{{cookiecutter.project_slug}}/config/settings/production.py index 236c6c9f..26fa6294 100644 --- a/{{cookiecutter.project_slug}}/config/settings/production.py +++ b/{{cookiecutter.project_slug}}/config/settings/production.py @@ -69,7 +69,8 @@ SECURE_CONTENT_TYPE_NOSNIFF = env.bool( # STORAGES # ------------------------------------------------------------------------------ # https://django-storages.readthedocs.io/en/latest/#installation -INSTALLED_APPS += ["storages"] # noqa F405 +INSTALLED_APPS += ['storages'] # noqa F405 +{% if cookiecutter.cloud_provider == 'AWS' %} # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings AWS_ACCESS_KEY_ID = env("DJANGO_AWS_ACCESS_KEY_ID") # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings @@ -88,22 +89,25 @@ AWS_S3_OBJECT_PARAMETERS = { AWS_DEFAULT_ACL = None # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings AWS_S3_REGION_NAME = env("DJANGO_AWS_S3_REGION_NAME", default=None) +{% elif cookiecutter.cloud_provider == 'GCE' %} +DEFAULT_FILE_STORAGE = 'storages.backends.gcloud.GoogleCloudStorage' +GS_BUCKET_NAME = env('DJANGO_GCE_STORAGE_BUCKET_NAME') +GS_DEFAULT_ACL = 'publicRead' +{% endif %} # STATIC # ------------------------ {% if cookiecutter.use_whitenoise == 'y' -%} STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" -{%- else %} -STATICFILES_STORAGE = "config.settings.production.StaticRootS3Boto3Storage" -STATIC_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/static/" +{%- endif -%} +{%- if cookiecutter.cloud_provider == 'AWS' %} +STATICFILES_STORAGE = 'config.settings.production.StaticRootS3Boto3Storage' +STATIC_URL = f'https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/static/' {%- endif %} # MEDIA # ------------------------------------------------------------------------------ -{% if cookiecutter.use_whitenoise == 'y' -%} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -MEDIA_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/" -{%- else %} +{%- if cookiecutter.cloud_provider == 'AWS' %} # region http://stackoverflow.com/questions/10390244/ # Full-fledge class: https://stackoverflow.com/a/18046120/104731 from storages.backends.s3boto3 import S3Boto3Storage # noqa E402 @@ -121,6 +125,9 @@ class MediaRootS3Boto3Storage(S3Boto3Storage): # endregion DEFAULT_FILE_STORAGE = "config.settings.production.MediaRootS3Boto3Storage" MEDIA_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/media/" +{%- elif cookiecutter.cloud_provider == 'GCE' %} +MEDIA_URL = 'https://storage.googleapis.com/{}/'.format(GS_BUCKET_NAME) +MEDIA_ROOT = 'https://storage.googleapis.com/{}/'.format(GS_BUCKET_NAME) {%- endif %} # TEMPLATES diff --git a/{{cookiecutter.project_slug}}/requirements/production.txt b/{{cookiecutter.project_slug}}/requirements/production.txt index 5a40f07a..b1758194 100644 --- a/{{cookiecutter.project_slug}}/requirements/production.txt +++ b/{{cookiecutter.project_slug}}/requirements/production.txt @@ -13,5 +13,9 @@ sentry-sdk==0.7.9 # https://github.com/getsentry/sentry-python # Django # ------------------------------------------------------------------------------ +{%- if cookiecutter.cloud_provider == 'AWS' %} django-storages[boto3]==1.7.1 # https://github.com/jschneier/django-storages +{%- elif cookiecutter.cloud_provider == 'GCE' %} +django-storages[google]==1.7.1 # https://github.com/jschneier/django-storages +{%- endif %} django-anymail[mailgun]==6.0 # https://github.com/anymail/django-anymail From 749920e81cd4e382b06cd1bcb32a17201fe44853 Mon Sep 17 00:00:00 2001 From: browniebroke Date: Sat, 6 Apr 2019 13:00:29 +0200 Subject: [PATCH 11/25] Update pytest_cases from 1.6.1 to 1.6.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2b2c55f3..25b0135b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,6 @@ flake8==3.7.6 # ------------------------------------------------------------------------------ tox==3.8.6 pytest==4.4.0 -pytest_cases==1.6.1 +pytest_cases==1.6.2 pytest-cookies==0.3.0 pyyaml==5.1 From c582ac48ec5bc61a849b3ceb6d5d4b6f0c854f7c Mon Sep 17 00:00:00 2001 From: browniebroke Date: Sat, 6 Apr 2019 13:00:32 +0200 Subject: [PATCH 12/25] Update sentry-sdk from 0.7.9 to 0.7.10 --- {{cookiecutter.project_slug}}/requirements/production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/production.txt b/{{cookiecutter.project_slug}}/requirements/production.txt index 5a40f07a..ea4b376f 100644 --- a/{{cookiecutter.project_slug}}/requirements/production.txt +++ b/{{cookiecutter.project_slug}}/requirements/production.txt @@ -8,7 +8,7 @@ psycopg2==2.8 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 Collectfast==0.6.2 # https://github.com/antonagestam/collectfast {%- endif %} {%- if cookiecutter.use_sentry == "y" %} -sentry-sdk==0.7.9 # https://github.com/getsentry/sentry-python +sentry-sdk==0.7.10 # https://github.com/getsentry/sentry-python {%- endif %} # Django From b5e030b78439c1118d8e64c1ee582bcb6595059e Mon Sep 17 00:00:00 2001 From: anuj Date: Sun, 7 Apr 2019 00:34:22 +0530 Subject: [PATCH 13/25] Corrected quotes, added parameter in tests --- tests/test_cookiecutter_generation.py | 4 ++++ .../config/settings/production.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_cookiecutter_generation.py b/tests/test_cookiecutter_generation.py index 36395627..17375b1c 100755 --- a/tests/test_cookiecutter_generation.py +++ b/tests/test_cookiecutter_generation.py @@ -11,6 +11,7 @@ PATTERN = "{{(\s?cookiecutter)[.](.*?)}}" RE_OBJ = re.compile(PATTERN) YN_CHOICES = ["y", "n"] +CLOUD_CHOICES = ["AWS", "GCE"] @pytest.fixture @@ -35,6 +36,7 @@ def context(): @pytest.mark.parametrize("use_sentry", YN_CHOICES, ids=lambda yn: f"sentry:{yn}") @pytest.mark.parametrize("use_compressor", YN_CHOICES, ids=lambda yn: f"cmpr:{yn}") @pytest.mark.parametrize("use_whitenoise", YN_CHOICES, ids=lambda yn: f"wnoise:{yn}") +@pytest.mark.parametrize("cloud_provider", CLOUD_CHOICES, ids=lambda yn: f"cloud:{yn}") def context_combination( windows, use_docker, @@ -43,6 +45,7 @@ def context_combination( use_sentry, use_compressor, use_whitenoise, + cloud_provider, ): """Fixture that parametrize the function where it's used.""" return { @@ -53,6 +56,7 @@ def context_combination( "use_mailhog": use_mailhog, "use_sentry": use_sentry, "use_whitenoise": use_whitenoise, + "cloud_provider": cloud_provider, } diff --git a/{{cookiecutter.project_slug}}/config/settings/production.py b/{{cookiecutter.project_slug}}/config/settings/production.py index 26fa6294..61f90104 100644 --- a/{{cookiecutter.project_slug}}/config/settings/production.py +++ b/{{cookiecutter.project_slug}}/config/settings/production.py @@ -69,7 +69,7 @@ SECURE_CONTENT_TYPE_NOSNIFF = env.bool( # STORAGES # ------------------------------------------------------------------------------ # https://django-storages.readthedocs.io/en/latest/#installation -INSTALLED_APPS += ['storages'] # noqa F405 +INSTALLED_APPS += ["storages"] # noqa F405 {% if cookiecutter.cloud_provider == 'AWS' %} # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings AWS_ACCESS_KEY_ID = env("DJANGO_AWS_ACCESS_KEY_ID") @@ -90,9 +90,9 @@ AWS_DEFAULT_ACL = None # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings AWS_S3_REGION_NAME = env("DJANGO_AWS_S3_REGION_NAME", default=None) {% elif cookiecutter.cloud_provider == 'GCE' %} -DEFAULT_FILE_STORAGE = 'storages.backends.gcloud.GoogleCloudStorage' -GS_BUCKET_NAME = env('DJANGO_GCE_STORAGE_BUCKET_NAME') -GS_DEFAULT_ACL = 'publicRead' +DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +GS_BUCKET_NAME = env("DJANGO_GCE_STORAGE_BUCKET_NAME") +GS_DEFAULT_ACL = "publicRead" {% endif %} # STATIC @@ -101,8 +101,8 @@ GS_DEFAULT_ACL = 'publicRead' STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" {%- endif -%} {%- if cookiecutter.cloud_provider == 'AWS' %} -STATICFILES_STORAGE = 'config.settings.production.StaticRootS3Boto3Storage' -STATIC_URL = f'https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/static/' +STATICFILES_STORAGE = "config.settings.production.StaticRootS3Boto3Storage" +STATIC_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/static/" {%- endif %} # MEDIA @@ -126,8 +126,8 @@ class MediaRootS3Boto3Storage(S3Boto3Storage): DEFAULT_FILE_STORAGE = "config.settings.production.MediaRootS3Boto3Storage" MEDIA_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/media/" {%- elif cookiecutter.cloud_provider == 'GCE' %} -MEDIA_URL = 'https://storage.googleapis.com/{}/'.format(GS_BUCKET_NAME) -MEDIA_ROOT = 'https://storage.googleapis.com/{}/'.format(GS_BUCKET_NAME) +MEDIA_URL = "https://storage.googleapis.com/{}/".format(GS_BUCKET_NAME) +MEDIA_ROOT = "https://storage.googleapis.com/{}/".format(GS_BUCKET_NAME) {%- endif %} # TEMPLATES From 2b141e646434396dbb40a2accd224c42dc105ce3 Mon Sep 17 00:00:00 2001 From: anuj Date: Sun, 7 Apr 2019 00:54:46 +0530 Subject: [PATCH 14/25] Add support for GCE static --- {{cookiecutter.project_slug}}/config/settings/production.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/config/settings/production.py b/{{cookiecutter.project_slug}}/config/settings/production.py index 61f90104..4adbdb9c 100644 --- a/{{cookiecutter.project_slug}}/config/settings/production.py +++ b/{{cookiecutter.project_slug}}/config/settings/production.py @@ -103,6 +103,8 @@ STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" {%- if cookiecutter.cloud_provider == 'AWS' %} STATICFILES_STORAGE = "config.settings.production.StaticRootS3Boto3Storage" STATIC_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/static/" +{%- elif cookiecutter.cloud_provider == 'GCE' %} +STATIC_URL = "https://storage.googleapis.com/{}/static".format(GS_BUCKET_NAME) {%- endif %} # MEDIA @@ -126,8 +128,8 @@ class MediaRootS3Boto3Storage(S3Boto3Storage): DEFAULT_FILE_STORAGE = "config.settings.production.MediaRootS3Boto3Storage" MEDIA_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com/media/" {%- elif cookiecutter.cloud_provider == 'GCE' %} -MEDIA_URL = "https://storage.googleapis.com/{}/".format(GS_BUCKET_NAME) -MEDIA_ROOT = "https://storage.googleapis.com/{}/".format(GS_BUCKET_NAME) +MEDIA_URL = "https://storage.googleapis.com/{}/media".format(GS_BUCKET_NAME) +MEDIA_ROOT = "https://storage.googleapis.com/{}/media".format(GS_BUCKET_NAME) {%- endif %} # TEMPLATES From 020933d336c09a7946edd69c92c2562f085c2ff2 Mon Sep 17 00:00:00 2001 From: browniebroke Date: Tue, 9 Apr 2019 13:00:39 +0200 Subject: [PATCH 15/25] Update sphinx from 2.0.0 to 2.0.1 --- {{cookiecutter.project_slug}}/requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index eb81e7d0..1c90cd84 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -2,7 +2,7 @@ Werkzeug==0.15.2 # https://github.com/pallets/werkzeug ipdb==0.12 # https://github.com/gotcha/ipdb -Sphinx==2.0.0 # https://github.com/sphinx-doc/sphinx +Sphinx==2.0.1 # https://github.com/sphinx-doc/sphinx {%- if cookiecutter.use_docker == 'y' %} psycopg2==2.8 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 {%- else %} From 6d906721fc74593c56b70a10ad9804c5ea6afb32 Mon Sep 17 00:00:00 2001 From: browniebroke Date: Wed, 10 Apr 2019 13:00:32 +0200 Subject: [PATCH 16/25] Update pytz from 2018.9 to 2019.1 --- {{cookiecutter.project_slug}}/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/base.txt b/{{cookiecutter.project_slug}}/requirements/base.txt index 9d4aec7d..39bec2d4 100644 --- a/{{cookiecutter.project_slug}}/requirements/base.txt +++ b/{{cookiecutter.project_slug}}/requirements/base.txt @@ -1,4 +1,4 @@ -pytz==2018.9 # https://github.com/stub42/pytz +pytz==2019.1 # https://github.com/stub42/pytz python-slugify==3.0.2 # https://github.com/un33k/python-slugify Pillow==6.0.0 # https://github.com/python-pillow/Pillow {%- if cookiecutter.use_compressor == "y" %} From 56aa332e0146ef94306ebac3f0fdacbd64fcb4fc Mon Sep 17 00:00:00 2001 From: Vlad Date: Sun, 14 Apr 2019 11:43:05 -0500 Subject: [PATCH 17/25] Documented Google Cloud Storage integration as an alternative to AWS S3 --- README.rst | 8 ++++++-- docs/project-generation-options.rst | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2c037e51..fccbdaf9 100644 --- a/README.rst +++ b/README.rst @@ -47,7 +47,7 @@ Features * Comes with custom user model ready to go * Optional custom static build using Gulp and livereload * Send emails via Anymail_ (using Mailgun_ by default, but switchable) -* Media storage using Amazon S3 +* Media storage using Amazon S3 or Google Cloud Storage * Docker support using docker-compose_ for development and production (using Traefik_ with LetsEncrypt_ support) * Procfile_ for deploying to Heroku * Instructions for deploying to PythonAnywhere_ @@ -62,7 +62,7 @@ Optional Integrations *These features can be enabled during initial project setup.* -* Serve static files from Amazon S3 or Whitenoise_ +* Serve static files from Amazon S3, Google Cloud Storage or Whitenoise_ * Configuration for Celery_ and Flower_ (the latter in Docker setup only) * Integration with MailHog_ for local email testing * Integration with Sentry_ for error logging @@ -181,6 +181,10 @@ Answer the prompts with your own desired options_. For example:: 1 - None 2 - Gulp Choose from 1, 2 [1]: 1 + Select cloud_provider: + 1 - AWS + 2 - GCS + Choose from 1, 2 [1]: 1 custom_bootstrap_compilation [n]: n Select open_source_license: 1 - MIT diff --git a/docs/project-generation-options.rst b/docs/project-generation-options.rst index a5483797..82e9edc0 100644 --- a/docs/project-generation-options.rst +++ b/docs/project-generation-options.rst @@ -63,6 +63,12 @@ js_task_runner: 1. None 2. Gulp_ +cloud_provider: + Select a cloud provider for static files. The choices are: + + 1. AWS_ + 2. GCS_ + custom_bootstrap_compilation: Indicates whether the project should support Bootstrap recompilation via the selected JavaScript task runner's task. This can be useful From 7c8a473460f6c90a9773357aaec955b4591f5f65 Mon Sep 17 00:00:00 2001 From: Vlad Date: Sun, 14 Apr 2019 11:57:48 -0500 Subject: [PATCH 18/25] Updated settings.rst to reflect new GS env variable --- docs/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/settings.rst b/docs/settings.rst index 212527cf..dd698a0d 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -45,6 +45,7 @@ DJANGO_AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID n/a DJANGO_AWS_SECRET_ACCESS_KEY AWS_SECRET_ACCESS_KEY n/a raises error DJANGO_AWS_STORAGE_BUCKET_NAME AWS_STORAGE_BUCKET_NAME n/a raises error DJANGO_AWS_S3_REGION_NAME AWS_S3_REGION_NAME n/a None +DJANGO_GCE_STORAGE_BUCKET_NAME GS_BUCKET_NAME n/a raises error SENTRY_DSN SENTRY_DSN n/a raises error DJANGO_SENTRY_LOG_LEVEL SENTRY_LOG_LEVEL n/a logging.INFO MAILGUN_API_KEY MAILGUN_ACCESS_KEY n/a raises error From c041453bef94930c658edffe3f009b799d459a7d Mon Sep 17 00:00:00 2001 From: Vlad Date: Sun, 14 Apr 2019 12:07:37 -0500 Subject: [PATCH 19/25] Updated project-generation-options.rst to provide helpful links in case a user needs to decide between AWS s3 and GCS and added myself to contributors.rst --- CONTRIBUTORS.rst | 2 ++ docs/project-generation-options.rst | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 9486b3a0..ded1606b 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -177,6 +177,7 @@ Listed in alphabetical order. Travis McNeill `@Travistock`_ @tavistock_esq Tubo Shi `@Tubo`_ Umair Ashraf `@umrashrf`_ @fabumair + Vlad Doster `@vladdoster`_ Vitaly Babiy Vivian Guillen `@viviangb`_ Will Farley `@goldhand`_ @g01dhand @@ -315,6 +316,7 @@ Listed in alphabetical order. .. _@mrcoles: https://github.com/mrcoles .. _@ericgroom: https://github.com/ericgroom .. _@hanaquadara: https://github.com/hanaquadara +.. _@vladdoster: https://github.com/vladdoster Special Thanks ~~~~~~~~~~~~~~ diff --git a/docs/project-generation-options.rst b/docs/project-generation-options.rst index 82e9edc0..470ef949 100644 --- a/docs/project-generation-options.rst +++ b/docs/project-generation-options.rst @@ -121,6 +121,9 @@ debug: .. _Gulp: https://github.com/gulpjs/gulp +.. _AWS: https://aws.amazon.com/s3/ +.. _GCS: https://cloud.google.com/storage/ + .. _Django Compressor: https://github.com/django-compressor/django-compressor .. _Celery: https://github.com/celery/celery From 9137bb561fc1a7fbe83a6ca61e5c0fffe0c10146 Mon Sep 17 00:00:00 2001 From: browniebroke Date: Tue, 16 Apr 2019 12:00:29 +0100 Subject: [PATCH 20/25] Update pytest from 4.4.0 to 4.4.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 25b0135b..cfd0c847 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ flake8==3.7.6 # Testing # ------------------------------------------------------------------------------ tox==3.8.6 -pytest==4.4.0 +pytest==4.4.1 pytest_cases==1.6.2 pytest-cookies==0.3.0 pyyaml==5.1 From 026dd6ed575d53d2af49b0878b8080b793c388c0 Mon Sep 17 00:00:00 2001 From: browniebroke Date: Tue, 16 Apr 2019 12:00:30 +0100 Subject: [PATCH 21/25] Update pytest from 4.4.0 to 4.4.1 --- {{cookiecutter.project_slug}}/requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index 1c90cd84..60c68dd1 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -12,7 +12,7 @@ psycopg2-binary==2.8 # https://github.com/psycopg/psycopg2 # Testing # ------------------------------------------------------------------------------ mypy==0.700 # https://github.com/python/mypy -pytest==4.4.0 # https://github.com/pytest-dev/pytest +pytest==4.4.1 # https://github.com/pytest-dev/pytest pytest-sugar==0.9.2 # https://github.com/Frozenball/pytest-sugar # Code quality From f6cac35b8743407b788ea98b8b2c0444ed25df04 Mon Sep 17 00:00:00 2001 From: Bruno Alla Date: Tue, 16 Apr 2019 11:23:41 -0500 Subject: [PATCH 22/25] Update docs/project-generation-options.rst Clearer diction describing cloud provider roles Co-Authored-By: vladdoster --- docs/project-generation-options.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/project-generation-options.rst b/docs/project-generation-options.rst index 470ef949..c9b3dff8 100644 --- a/docs/project-generation-options.rst +++ b/docs/project-generation-options.rst @@ -64,7 +64,7 @@ js_task_runner: 2. Gulp_ cloud_provider: - Select a cloud provider for static files. The choices are: + Select a cloud provider for static & media files. The choices are: 1. AWS_ 2. GCS_ From cd13ab4c823e3c9619c9deac5ca368b9af477a9a Mon Sep 17 00:00:00 2001 From: Bruno Alla Date: Tue, 16 Apr 2019 11:30:22 -0500 Subject: [PATCH 23/25] Update docs/settings.rst Documented GOOGLE_APPLICATION_CREDENTIALS env var Co-Authored-By: vladdoster --- docs/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/settings.rst b/docs/settings.rst index dd698a0d..1830a47c 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -46,6 +46,7 @@ DJANGO_AWS_SECRET_ACCESS_KEY AWS_SECRET_ACCESS_KEY n/a DJANGO_AWS_STORAGE_BUCKET_NAME AWS_STORAGE_BUCKET_NAME n/a raises error DJANGO_AWS_S3_REGION_NAME AWS_S3_REGION_NAME n/a None DJANGO_GCE_STORAGE_BUCKET_NAME GS_BUCKET_NAME n/a raises error +GOOGLE_APPLICATION_CREDENTIALS n/a n/a raises error SENTRY_DSN SENTRY_DSN n/a raises error DJANGO_SENTRY_LOG_LEVEL SENTRY_LOG_LEVEL n/a logging.INFO MAILGUN_API_KEY MAILGUN_ACCESS_KEY n/a raises error From 6f2a652c2b5be918a67294522e7fdeb04924b539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20C=2E=20Barrionuevo=20da=20Luz?= Date: Tue, 16 Apr 2019 19:42:57 -0300 Subject: [PATCH 24/25] update PostgreSQL versions in the example --- README.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index fccbdaf9..5a0fc5fb 100644 --- a/README.rst +++ b/README.rst @@ -169,13 +169,15 @@ Answer the prompts with your own desired options_. For example:: use_heroku [n]: y use_compressor [n]: y Select postgresql_version: - 1 - 10.3 - 2 - 10.2 - 3 - 10.1 - 4 - 9.6 - 5 - 9.5 - 6 - 9.4 - 7 - 9.3 + 1 - 10.5 + 2 - 10.4 + 3 - 10.3 + 4 - 10.2 + 5 - 10.1 + 6 - 9.6 + 7 - 9.5 + 8 - 9.4 + 9 - 9.3 Choose from 1, 2, 3, 4 [1]: 1 Select js_task_runner: 1 - None From 4d8e3c8c137a9bd2e54a4a408d27ee0e65e9f57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20C=2E=20Barrionuevo=20da=20Luz?= Date: Tue, 16 Apr 2019 20:24:05 -0300 Subject: [PATCH 25/25] update "description" in the example --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5a0fc5fb..91fbe3ef 100644 --- a/README.rst +++ b/README.rst @@ -155,7 +155,7 @@ Answer the prompts with your own desired options_. For example:: project_slug [reddit_clone]: reddit author_name [Daniel Roy Greenfeld]: Daniel Greenfeld email [you@example.com]: pydanny@gmail.com - description [A short description of the project.]: A reddit clone. + description [Behold My Awesome Project!]: A reddit clone. domain_name [example.com]: myreddit.com version [0.1.0]: 0.0.1 timezone [UTC]: America/Los_Angeles