From cbb0e19de75eed12cfa84ca4e6d6e8a34ebe926e Mon Sep 17 00:00:00 2001 From: Bruno Alla Date: Sun, 29 Jan 2023 12:12:12 +0000 Subject: [PATCH] Add Webpack support (#3623) * Add support for Webpack as frontend pipeline * Rename CI jobs * Fix a couple of issues with Webpack + Docker * Don't include Boostrap CSS from CDN with Webpack * Rename variable * Set publicPath in prod webpack config * Fix removal of SASS files in post-gen hooks * Add Webpack to readme usage section * Run Django + Webpack dev server concurrently without Docker * Fix async runserver command with Gulp/Webpack * Upgrade django-webpack-loader to 1.5.0 * Pass variables required by Webpack at build time * Upgrade django-webpack-loader to 1.7.0 * Add missing condition * Add support for Azure Storage + Webpack * Whitespaces * Rename ROOT_DIR -> BASE_DIR * Rename jobs * Bump django-webpack-loader to latest * Document limitation of Docker + Webpack + no Whitenoise * Update section on custom Bootstrap compilation in generated readme --- .github/workflows/ci.yml | 22 ++-- README.md | 3 +- cookiecutter.json | 3 +- docs/deployment-on-heroku.rst | 8 +- docs/deployment-with-docker.rst | 26 ++++ docs/developing-locally.rst | 2 +- docs/project-generation-options.rst | 6 +- docs/troubleshooting.rst | 12 +- hooks/post_gen_project.py | 111 +++++++++++++++++- tests/test_bare.sh | 6 +- tests/test_cookiecutter_generation.py | 1 + tests/test_docker.sh | 6 + {{cookiecutter.project_slug}}/.gitignore | 4 + .../docker_compose_up_django.xml | 2 +- .../.idea/{{cookiecutter.project_slug}}.iml | 2 +- {{cookiecutter.project_slug}}/README.md | 7 +- .../compose/production/django/Dockerfile | 18 ++- .../config/settings/base.py | 16 +++ .../config/settings/local.py | 8 +- {{cookiecutter.project_slug}}/gulpfile.js | 4 +- {{cookiecutter.project_slug}}/local.yml | 4 +- {{cookiecutter.project_slug}}/package.json | 29 ++++- {{cookiecutter.project_slug}}/production.yml | 13 ++ .../requirements/base.txt | 7 +- .../webpack/common.config.js | 55 +++++++++ .../webpack/dev.config.js | 20 ++++ .../webpack/prod.config.js | 28 +++++ .../static/js/project.js | 4 + .../static/js/vendors.js | 2 + .../static/sass/project.scss | 2 +- .../templates/base.html | 17 ++- 31 files changed, 399 insertions(+), 49 deletions(-) create mode 100644 {{cookiecutter.project_slug}}/webpack/common.config.js create mode 100644 {{cookiecutter.project_slug}}/webpack/dev.config.js create mode 100644 {{cookiecutter.project_slug}}/webpack/prod.config.js create mode 100644 {{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/static/js/vendors.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d466f403..08ac54d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: - windows-latest - macOS-latest - name: "Run tests" + name: "pytest ${{ matrix.os }}" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -49,10 +49,14 @@ jobs: script: - name: Basic args: "" - - name: Extended - args: "use_celery=y use_drf=y frontend_pipeline=Gulp" + - name: Celery & DRF + args: "use_celery=y use_drf=y" + - name: Gulp + args: "frontend_pipeline=Gulp" + - name: Webpack + args: "frontend_pipeline=Webpack" - name: "${{ matrix.script.name }} Docker" + name: "Docker ${{ matrix.script.name }}" runs-on: ubuntu-latest env: DOCKER_BUILDKIT: 1 @@ -74,12 +78,14 @@ jobs: fail-fast: false matrix: script: - - name: With Celery + - name: Celery args: "use_celery=y frontend_pipeline='Django Compressor'" - - name: With Gulp - args: "frontend_pipeline='Gulp'" + - name: Gulp + args: "frontend_pipeline=Gulp" + - name: Webpack + args: "frontend_pipeline=Webpack" - name: "${{ matrix.script.name }} Bare metal" + name: "Bare metal ${{ matrix.script.name }}" runs-on: ubuntu-latest services: redis: diff --git a/README.md b/README.md index d8fc58ec..0697862c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ production-ready Django projects quickly. - Registration via [django-allauth](https://github.com/pennersr/django-allauth) - Comes with custom user model ready to go - Optional basic ASGI setup for Websockets -- Optional custom static build using Gulp and livereload +- Optional custom static build using Gulp or Webpack - Send emails via [Anymail](https://github.com/anymail/django-anymail) (using [Mailgun](http://www.mailgun.com/) by default or Amazon SES if AWS is selected cloud provider, but switchable) - Media storage using Amazon S3, Google Cloud Storage or Azure Storage - Docker support using [docker-compose](https://github.com/docker/compose) for development and production (using [Traefik](https://traefik.io/) with [LetsEncrypt](https://letsencrypt.org/) support) @@ -149,6 +149,7 @@ Answer the prompts with your own desired [options](http://cookiecutter-django.re 1 - None 2 - Django Compressor 3 - Gulp + 4 - Webpack Choose from 1, 2, 3, 4 [1]: 1 use_celery [n]: y use_mailhog [n]: n diff --git a/cookiecutter.json b/cookiecutter.json index 97041101..cf4da9a4 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -46,7 +46,8 @@ "frontend_pipeline": [ "None", "Django Compressor", - "Gulp" + "Gulp", + "Webpack" ], "use_celery": "n", "use_mailhog": "n", diff --git a/docs/deployment-on-heroku.rst b/docs/deployment-on-heroku.rst index 71fb45dd..71c6e11b 100644 --- a/docs/deployment-on-heroku.rst +++ b/docs/deployment-on-heroku.rst @@ -109,10 +109,10 @@ Or add the DSN for your account, if you already have one: .. _Sentry add-on: https://elements.heroku.com/addons/sentry -Gulp & Bootstrap compilation -++++++++++++++++++++++++++++ +Gulp or Webpack ++++++++++++++++ -If you've opted for Gulp, you'll most likely need to setup +If you've opted for Gulp or Webpack as frontend pipeline, you'll most likely need to setup your app to use `multiple buildpacks`_: one for Python & one for Node.js: .. code-block:: bash @@ -121,7 +121,7 @@ your app to use `multiple buildpacks`_: one for Python & one for Node.js: At time of writing, this should do the trick: during deployment, the Heroku should run ``npm install`` and then ``npm build``, -which runs Gulp in cookiecutter-django. +which run the SASS compilation & JS bundling. If things don't work, please refer to the Heroku docs. diff --git a/docs/deployment-with-docker.rst b/docs/deployment-with-docker.rst index fcce7e6f..a431679b 100644 --- a/docs/deployment-with-docker.rst +++ b/docs/deployment-with-docker.rst @@ -84,6 +84,32 @@ You can read more about this feature and how to configure it, at `Automatic HTTP .. _Automatic HTTPS: https://docs.traefik.io/https/acme/ +.. _webpack-whitenoise-limitation: + +Webpack without Whitenoise limitation +------------------------------------- + +If you opt for Webpack without Whitenoise, Webpack needs to know the static URL at build time, when running ``docker-compose build`` (See ``webpack/prod.config.js``). Depending on your setup, this URL may come from the following environment variables: + +- ``AWS_STORAGE_BUCKET_NAME`` +- ``DJANGO_AWS_S3_CUSTOM_DOMAIN`` +- ``DJANGO_GCP_STORAGE_BUCKET_NAME`` +- ``DJANGO_AZURE_CONTAINER_NAME`` + +The Django settings are getting these values at runtime via the ``.envs/.production/.django`` file , but Docker does not read this file at build time, it only look for a ``.env`` in the root of the project. Failing to pass the values correctly will result in a page without CSS styles nor javascript. + +To solve this, you can either: + +1. merge all the env files into ``.env`` by running:: + + merge_production_dotenvs_in_dotenv.py + +2. create a ``.env`` file in the root of the project with just variables you need. You'll need to also define them in ``.envs/.production/.django`` (hence duplicating them). +3. set these variables when running the build command:: + + DJANGO_AWS_S3_CUSTOM_DOMAIN=example.com docker-compose -f production.yml build``. + +None of these options are ideal, we're open to suggestions on how to improve this. If you think you have one, please open an issue or a pull request. (Optional) Postgres Data Volume Modifications --------------------------------------------- diff --git a/docs/developing-locally.rst b/docs/developing-locally.rst index 2b943805..fb66536f 100644 --- a/docs/developing-locally.rst +++ b/docs/developing-locally.rst @@ -172,7 +172,7 @@ You can also use Django admin to queue up tasks, thanks to the `django-celerybea Sass Compilation & Live Reloading --------------------------------- -If you've opted for Gulp as front-end pipeline, the project comes configured with `Sass`_ compilation and `live reloading`_. As you change you Sass/JS source files, the task runner will automatically rebuild the corresponding CSS and JS assets and reload them in your browser without refreshing the page. +If you've opted for Gulp or Webpack as front-end pipeline, the project comes configured with `Sass`_ compilation and `live reloading`_. As you change you Sass/JS source files, the task runner will automatically rebuild the corresponding CSS and JS assets and reload them in your browser without refreshing the page. #. Make sure that `Node.js`_ v16 is installed on your machine. #. In the project root, install the JS dependencies with:: diff --git a/docs/project-generation-options.rst b/docs/project-generation-options.rst index 0560badd..d4ad8a9a 100644 --- a/docs/project-generation-options.rst +++ b/docs/project-generation-options.rst @@ -95,7 +95,10 @@ frontend_pipeline: 1. None 2. `Django Compressor`_ - 3. `Gulp`_: support Bootstrap recompilation with real-time variables alteration. + 3. `Gulp`_ + 4. `Webpack`_ + +Both Gulp and Webpack support Bootstrap recompilation with real-time variables alteration. use_celery: Indicates whether the project should be configured to use Celery_. @@ -145,6 +148,7 @@ debug: .. _PostgreSQL: https://www.postgresql.org/docs/ .. _Gulp: https://github.com/gulpjs/gulp +.. _Webpack: https://webpack.js.org .. _AWS: https://aws.amazon.com/s3/ .. _GCP: https://cloud.google.com/storage/ diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index ba8ab53e..293e9b65 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -1,5 +1,5 @@ Troubleshooting -===================================== +=============== This page contains some advice about errors and problems commonly encountered during the development of Cookiecutter Django applications. @@ -38,6 +38,16 @@ To fix this, you can either: .. _rm: https://docs.docker.com/engine/reference/commandline/volume_rm/ .. _prune: https://docs.docker.com/v17.09/engine/reference/commandline/system_prune/ +Variable is not set. Defaulting to a blank string +------------------------------------------------- + +Example:: + + WARN[0000] The "DJANGO_AWS_STORAGE_BUCKET_NAME" variable is not set. Defaulting to a blank string. + WARN[0000] The "DJANGO_AWS_S3_CUSTOM_DOMAIN" variable is not set. Defaulting to a blank string. + +You have probably opted for Docker + Webpack without Whitenoise. This is a know limitation of the combination, which needs a little bit of manual intervention. See the :ref:`dedicated section about it `. + Others ------ diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 5655b61f..b64bbbaf 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -10,6 +10,7 @@ TODO: restrict Cookiecutter Django project initialization to """ from __future__ import print_function +import json import os import random import shutil @@ -87,15 +88,30 @@ def remove_heroku_build_hooks(): shutil.rmtree("bin") +def remove_sass_files(): + shutil.rmtree(os.path.join("{{cookiecutter.project_slug}}", "static", "sass")) + + def remove_gulp_files(): file_names = ["gulpfile.js"] for file_name in file_names: os.remove(file_name) - remove_sass_files() -def remove_sass_files(): - shutil.rmtree(os.path.join("{{cookiecutter.project_slug}}", "static", "sass")) +def remove_webpack_files(): + shutil.rmtree("webpack") + remove_vendors_js() + + +def remove_vendors_js(): + vendors_js_path = os.path.join( + "{{ cookiecutter.project_slug }}", + "static", + "js", + "vendors.js", + ) + if os.path.exists(vendors_js_path): + os.remove(vendors_js_path) def remove_packagejson_file(): @@ -104,6 +120,83 @@ def remove_packagejson_file(): os.remove(file_name) +def update_package_json(remove_dev_deps=None, remove_keys=None, scripts=None): + remove_dev_deps = remove_dev_deps or [] + remove_keys = remove_keys or [] + scripts = scripts or {} + with open("package.json", mode="r") as fd: + content = json.load(fd) + for package_name in remove_dev_deps: + content["devDependencies"].pop(package_name) + for key in remove_keys: + content.pop(key) + content["scripts"].update(scripts) + with open("package.json", mode="w") as fd: + json.dump(content, fd, ensure_ascii=False, indent=2) + fd.write("\n") + + +def handle_js_runner(choice, use_docker, use_async): + if choice == "Gulp": + update_package_json( + remove_dev_deps=[ + "@babel/core", + "@babel/preset-env", + "babel-loader", + "concurrently", + "css-loader", + "mini-css-extract-plugin", + "postcss-loader", + "postcss-preset-env", + "sass-loader", + "webpack", + "webpack-bundle-tracker", + "webpack-cli", + "webpack-dev-server", + "webpack-merge", + ], + remove_keys=["babel"], + scripts={ + "dev": "gulp", + "build": "gulp generate-assets", + }, + ) + remove_webpack_files() + elif choice == "Webpack": + scripts = { + "dev": "webpack serve --config webpack/dev.config.js", + "build": "webpack --config webpack/prod.config.js", + } + remove_dev_deps = [ + "browser-sync", + "cssnano", + "gulp", + "gulp-imagemin", + "gulp-plumber", + "gulp-postcss", + "gulp-rename", + "gulp-sass", + "gulp-uglify-es", + ] + if not use_docker: + dev_django_cmd = ( + "uvicorn config.asgi:application --reload" + if use_async + else "python manage.py runserver_plus" + ) + scripts.update( + { + "dev": "concurrently npm:dev:*", + "dev:webpack": "webpack serve --config webpack/dev.config.js", + "dev:django": dev_django_cmd, + } + ) + else: + remove_dev_deps.append("concurrently") + update_package_json(remove_dev_deps=remove_dev_deps, scripts=scripts) + remove_gulp_files() + + def remove_celery_files(): file_names = [ os.path.join("config", "celery_app.py"), @@ -384,13 +477,21 @@ def main(): if "{{ cookiecutter.keep_local_envs_in_vcs }}".lower() == "y": append_to_gitignore_file("!.envs/.local/") - if "{{ cookiecutter.frontend_pipeline }}" != "Gulp": + if "{{ cookiecutter.frontend_pipeline }}" in ["None", "Django Compressor"]: remove_gulp_files() + remove_webpack_files() + remove_sass_files() remove_packagejson_file() if "{{ cookiecutter.use_docker }}".lower() == "y": remove_node_dockerfile() + else: + handle_js_runner( + "{{ cookiecutter.frontend_pipeline }}", + use_docker=("{{ cookiecutter.use_docker }}".lower() == "y"), + use_async=("{{ cookiecutter.use_async }}".lower() == "y"), + ) - if "{{ cookiecutter.cloud_provider}}" == "None": + if "{{ cookiecutter.cloud_provider }}" == "None": print( WARNING + "You chose not to use a cloud provider, " "media files won't be served in production." + TERMINATOR diff --git a/tests/test_bare.sh b/tests/test_bare.sh index 05da9328..afd12fec 100755 --- a/tests/test_bare.sh +++ b/tests/test_bare.sh @@ -32,13 +32,11 @@ pytest # Make sure the check doesn't raise any warnings python manage.py check --fail-level WARNING +# Run npm build script if package.json is present if [ -f "package.json" ] then npm install - if [ -f "gulpfile.js" ] - then - npm run build - fi + npm run build fi # Generate the HTML for the documentation diff --git a/tests/test_cookiecutter_generation.py b/tests/test_cookiecutter_generation.py index 3a881cc6..5608bf88 100755 --- a/tests/test_cookiecutter_generation.py +++ b/tests/test_cookiecutter_generation.py @@ -101,6 +101,7 @@ SUPPORTED_COMBINATIONS = [ {"frontend_pipeline": "None"}, {"frontend_pipeline": "Django Compressor"}, {"frontend_pipeline": "Gulp"}, + {"frontend_pipeline": "Webpack"}, {"use_celery": "y"}, {"use_celery": "n"}, {"use_mailhog": "y"}, diff --git a/tests/test_docker.sh b/tests/test_docker.sh index b3663bd2..28d23289 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -41,3 +41,9 @@ docker-compose -f local.yml run django python manage.py check --fail-level WARNI # Generate the HTML for the documentation docker-compose -f local.yml run docs make html + +# Run npm build script if package.json is present +if [ -f "package.json" ] +then + docker-compose -f local.yml run node npm run build +fi diff --git a/{{cookiecutter.project_slug}}/.gitignore b/{{cookiecutter.project_slug}}/.gitignore index 17e9249c..19bb2bc0 100644 --- a/{{cookiecutter.project_slug}}/.gitignore +++ b/{{cookiecutter.project_slug}}/.gitignore @@ -348,3 +348,7 @@ vendors.js *.min.js *.min.js.map {%- endif %} +{%- if cookiecutter.frontend_pipeline == 'Webpack' %} +{{ cookiecutter.project_slug }}/static/webpack_bundles/ +webpack-stats.json +{%- endif %} diff --git a/{{cookiecutter.project_slug}}/.idea/runConfigurations/docker_compose_up_django.xml b/{{cookiecutter.project_slug}}/.idea/runConfigurations/docker_compose_up_django.xml index ad3b6a35..e84c5ffd 100644 --- a/{{cookiecutter.project_slug}}/.idea/runConfigurations/docker_compose_up_django.xml +++ b/{{cookiecutter.project_slug}}/.idea/runConfigurations/docker_compose_up_django.xml @@ -10,7 +10,7 @@