diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c2d32a..1f531a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,6 @@ version: 2 build_preconditions: &build_preconditions - requires: - - server - - vault - - script - - auditorium - - packages filters: branches: only: @@ -23,136 +17,6 @@ deploy_preconditions: &deploy_preconditions - development jobs: - server: - docker: - - image: circleci/golang:1.13 - environment: - POSTGRES_CONNECTION_STRING: postgres://circle:test@localhost:5432/circle_test?sslmode=disable - PORT: 8080 - EVENT_RETENTION_PERIOD: 4464h - COOKIE_EXCHANGE_SECRET: VswgMshC4mPDfey8o+yScg== - - image: circleci/postgres:11.2-alpine - environment: - - POSTGRES_USER=circle - - POSTGRES_PASSWORD=test - working_directory: ~/offen/server - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-server-{{ checksum "go.mod" }} - - run: - name: Download modules - command: go mod download - - save_cache: - paths: - - /go/pkg/mod - key: offen-server-{{ checksum "go.mod" }} - - run: - name: Waiting for Postgres to be ready - command: | - for i in `seq 1 10`; - do - nc -z localhost 5432 && echo Success && exit 0 - echo -n . - sleep 1 - done - echo Failed waiting for Postgres && exit 1 - - run: - name: Run tests - command: | - cp ~/offen/bootstrap.yml . - make test-ci - - vault: - docker: - - image: circleci/node:10-browsers - working_directory: ~/offen/vault - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-vault-{{ checksum "package.json" }} - - run: - name: Install dependencies - command: npm install - - save_cache: - paths: - - ~/offen/vault/node_modules - key: offen-vault-{{ checksum "package.json" }} - - run: - name: Run tests - command: npm test - - script: - docker: - - image: circleci/node:10-browsers - working_directory: ~/offen/script - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-script-{{ checksum "package.json" }} - - run: - name: Install lsof - command: sudo apt-get install lsof - - run: - name: Install dependencies - command: npm install - - save_cache: - paths: - - ~/offen/script/node_modules - key: offen-script-{{ checksum "package.json" }} - - run: - name: Run tests - command: npm test - - auditorium: - docker: - - image: circleci/node:10-browsers - working_directory: ~/offen/auditorium - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-auditorium-{{ checksum "package.json" }} - - run: - name: Install lsof - command: sudo apt-get install lsof - - run: - name: Install dependencies - command: npm install - - save_cache: - paths: - - ~/offen/auditorium/node_modules - key: offen-auditorium-{{ checksum "package.json" }} - - run: - name: Run tests - command: npm test - - packages: - docker: - - image: circleci/node:10-browsers - working_directory: ~/offen/packages - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-packages-{{ checksum "package.json" }} - - run: - name: Install lsof - command: sudo apt-get install lsof - - run: - name: Install dependencies - command: npm install - - save_cache: - paths: - - ~/offen/packages/node_modules - key: offen-packages-{{ checksum "package.json" }} - - run: - name: Run tests - command: npm test - build: docker: - image: docker:18-git @@ -165,13 +29,11 @@ jobs: - v1-{{ .Branch }} paths: - /caches/proxy.tar - - /caches/server.tar - run: name: Load Docker image layer cache command: | set +o pipefail docker load -i /caches/proxy.tar | true - docker load -i /caches/server.tar | true - run: name: Build application Docker image command: | @@ -184,19 +46,16 @@ jobs: export DOCKER_IMAGE_TAG="latest" export ROBOTS_FILE="robots.txt.staging" fi - docker build -t offen/server:$DOCKER_IMAGE_TAG -f build/server/Dockerfile . docker build --build-arg siteurl=$SITEURL --build-arg robots=$ROBOTS_FILE -t offen/proxy:$DOCKER_IMAGE_TAG -f build/proxy/Dockerfile . - run: name: Save Docker image layer cache command: | mkdir -p /caches docker save -o /caches/proxy.tar offen/proxy - docker save -o /caches/server.tar offen/server - save_cache: key: v1-{{ .Branch }}-{{ epoch }} paths: - /caches/proxy.tar - - /caches/server.tar - deploy: name: Push application Docker image command: | @@ -206,7 +65,6 @@ jobs: export DOCKER_IMAGE_TAG="latest" fi echo "$DOCKER_PASS" | docker login --username $DOCKER_USER --password-stdin - docker push offen/server:$DOCKER_IMAGE_TAG docker push offen/proxy:$DOCKER_IMAGE_TAG deploy: @@ -232,9 +90,8 @@ jobs: sudo pip install awsebcli --upgrade - run: name: Run database migrations - working_directory: ~/offen/server command: | - go run cmd/offen/main.go migrate -conn $(aws secretsmanager get-secret-value --secret-id $TARGET_ENVIRONMENT/postgresConnectionString | jq -r '.SecretString') + echo "TBD" - run: name: Deploy command: | @@ -243,13 +100,8 @@ jobs: workflows: version: 2 - test_build_deploy: + build_deploy: jobs: - - server - - vault - - script - - auditorium - - packages - build: <<: *build_preconditions - deploy: diff --git a/.gitignore b/.gitignore index 0503146..83affc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,118 @@ .vscode *.pem -bootstrap-alpha.yml Dockerrun.aws.json -node_modules/ -dist/ *.log -bin/ -statik.go +# built pelican site +output/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +test.py +*.json + diff --git a/Makefile b/Makefile index 2df78dd..aa5a312 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,6 @@ help: @echo " **IMPORTANT**: this wipes any existing data in your local database." @echo " build" @echo " Build the production images." - @echo " secret" - @echo " Generate a random base64 encoded secret" setup: dev-build update howto @@ -33,10 +31,6 @@ bootstrap: update: @echo "Installing / updating dependencies ..." - @docker-compose run script npm install - @docker-compose run vault npm install - @docker-compose run auditorium npm install - @docker-compose run server go mod download @docker-compose run homepage pip install --user -r requirements.txt DOCKER_IMAGE_TAG ?= latest @@ -46,7 +40,4 @@ build: @docker build -t offen/server:${DOCKER_IMAGE_TAG} -f build/server/Dockerfile . @docker build --build-arg siteurl=${SITEURL} --build-arg robots=${ROBOTS_FILE} -t offen/proxy:${DOCKER_IMAGE_TAG} -f build/proxy/Dockerfile . -secret: - @docker-compose run server make secret - .PHONY: setup build bootstrap build secret diff --git a/README.md b/README.md index 3ec5263..a0e37f4 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,13 @@ -# offen -[![CircleCI](https://circleci.com/gh/offen/offen/tree/master.svg?style=svg)](https://circleci.com/gh/offen/offen/tree/master) +# website +[![CircleCI](https://circleci.com/gh/offen/website/tree/master.svg?style=svg)](https://circleci.com/gh/offen/offen/tree/master) [![Patreon](https://img.shields.io/static/v1.svg?label=patreon&message=donate&color=e85b46)](https://www.patreon.com/offen) -> The offen analytics software +> The www.offen.dev website -This repository contains all source code needed to build and run __offen__, both on the server as well as on the client. See each of the READMEs in the subdirectories for instructions on how to work on that particular area of the application. +This repository contains the source code for the www.offen.dev website, as well as the code needed for running and extending an instance of `offen/offen`. --- -Development of __offen__ has just started, so instructions are rare and things will stay highly volatile for quite some while. - -Guidelines for running and developing the Software will be added when it makes sense to do so. If you have questions in the meantime, feel free to open an issue. - ### Developing the application The development setup requires `docker` and `docker-compose` to be installed. @@ -34,7 +30,7 @@ You can test your setup by starting the application: $ docker-compose up ``` -which should enable you to access and use the `auditorium` +which should enable you to access the homepage at and use the `auditorium` at ### License diff --git a/docker-compose.yml b/docker-compose.yml index a845eb4..d906c68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,23 +6,13 @@ services: volumes: - ./nginx.conf:/etc/nginx/nginx.conf ports: - - 8080:80 + - 8000:80 depends_on: - homepage - server - - auditorium - - vault - - script server: - build: - context: '.' - dockerfile: Dockerfile.golang - working_dir: /offen/server - volumes: - - .:/offen - - ./bootstrap.yml:/offen/server/bootstrap.yml - - serverdeps:/go/pkg/mod + image: offen/server:latest environment: POSTGRES_CONNECTION_STRING: postgres://postgres:develop@server_database:5432/postgres?sslmode=disable PORT: 8080 @@ -31,62 +21,25 @@ services: EVENT_RETENTION_PERIOD: 4464h ACCOUNT_USER_EMAIL_SALT: JuhbRA4lCdo8rt5qVdLpk3== SECURE_COOKIE: 'off' - command: refresh run - links: + depends_on: - server_database - ports: - - 8081:8080 server_database: image: postgres:11.2 environment: POSTGRES_PASSWORD: develop - vault: - build: - context: '.' - dockerfile: Dockerfile.node - working_dir: /offen/vault - volumes: - - .:/offen - - vaultdeps:/offen/vault/node_modules - command: npm start -- --port 9977 - - script: - build: - context: '.' - dockerfile: Dockerfile.node - working_dir: /offen/script - volumes: - - .:/offen - - scriptdeps:/offen/script/node_modules - command: npm start -- --port 9966 - - auditorium: - build: - context: '.' - dockerfile: Dockerfile.node - working_dir: /offen/auditorium - volumes: - - .:/offen - - auditoriumdeps:/offen/auditorium/node_modules - command: npm start -- --port 9955 - homepage: build: context: '.' dockerfile: ./Dockerfile.python - working_dir: /offen/homepage + working_dir: /website/homepage volumes: - - .:/offen + - .:/website - homepagedeps:/root/.local command: make devserver environment: DEBUG: 1 volumes: - serverdeps: - scriptdeps: - auditoriumdeps: - vaultdeps: homepagedeps: diff --git a/homepage/.gitignore b/homepage/.gitignore deleted file mode 100644 index 2a86b0d..0000000 --- a/homepage/.gitignore +++ /dev/null @@ -1,110 +0,0 @@ -# built pelican site -output/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ - -test.py -*.json diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..f7462f5 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,77 @@ +events {} + +http { + include mime.types; + + upstream homepage { + server homepage:8000; + } + + upstream server { + server server:8080; + } + + map $request_uri $expires { + default off; + "~-[0-9a-z]{10}\.js$" 1d; + "~*(woff|woff2|ttf|eot)$" 1d; + } + + map $sent_http_content_type $csp { + default ""; + "~^text/html" "default-src 'self' 'unsafe-eval'; style-src 'self' 'unsafe-inline'"; + } + + server { + listen 80; + expires $expires; + add_header Content-Security-Policy $csp; + add_header Referrer-Policy 'origin-when-cross-origin'; + add_header X-Content-Type-Options 'nosniff'; + add_header X-XSS-Protection '1; mode=block'; + + location /api/ { + proxy_pass http://server; + proxy_redirect off; + } + + location /vault/ { + proxy_pass http://server; + proxy_redirect off; + } + + location /auditorium/ { + proxy_pass http://server; + proxy_redirect off; + } + + location /script.js { + proxy_pass http://server; + proxy_redirect off; + } + + location /healthz { + proxy_pass http://server; + proxy_redirect off; + } + + location /favicon.ico { + root /www/data/theme/images; + } + + location / { + proxy_pass http://homepage; + proxy_redirect off; + } + + error_page 404 /custom_404.html; + # in development, this handles empty responses by the pelican dev server which are 404s in the end + error_page 502 /custom_404.html; + location = /custom_404.html { + internal; + proxy_pass http://homepage/404.html; + proxy_redirect off; + add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive" always; + } + } +}