From c21472128b6261570e51c73e88cc8ec0d826bd97 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Fri, 30 Aug 2019 21:02:06 +0200 Subject: [PATCH 01/20] scaffold single domain setup behind nginx --- docker-compose.yml | 85 ++++++++++------------------------------------ 1 file changed, 18 insertions(+), 67 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6de6f18..eb351d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,18 @@ version: '3' services: - kms: - build: - context: '.' - dockerfile: Dockerfile.golang - working_dir: /offen/kms + proxy: + image: nginx:1.17-alpine volumes: - - .:/offen - - kmsdeps:/go/pkg/mod - environment: - KEY_FILE: key.txt - PORT: 8081 - CORS_ORIGIN: http://localhost:9977 - JWT_PUBLIC_KEY: http://accounts:5000/api/key + - ./nginx.conf:/etc/nginx/nginx.conf + - ./styles:/code/styles ports: - - 8081:8081 - command: refresh run - links: - - accounts - - server_database: - image: postgres:11.2 - environment: - POSTGRES_PASSWORD: develop - - accounts_database: - image: mysql:5.7 - ports: - - "3306:3306" - environment: - MYSQL_DATABASE: mysql - MYSQL_ROOT_PASSWORD: develop + - 8080:80 + depends_on: + - server + - auditorium + - vault + - script server: build: @@ -52,13 +33,14 @@ services: DEVELOPMENT: '1' COOKIE_EXCHANGE_SECRET: Wsttdo4Z3mXV5sTc EVENT_RETENTION_PERIOD: 4464h - ports: - - 8080:8080 command: refresh run links: - server_database - depends_on: - - kms + + server_database: + image: postgres:11.2 + environment: + POSTGRES_PASSWORD: develop vault: build: @@ -69,13 +51,11 @@ services: - .:/offen - vaultdeps:/offen/vault/node_modules command: npm start -- --port 9977 - ports: - - 9977:9977 environment: - SERVER_HOST=http://localhost:8080 - KMS_HOST=http://localhost:8081 - SCRIPT_HOST=http://localhost:9977 - - AUDITORIUM_HOST=http://localhost:9955 + - AUDITORIUM_HOST=http://localhost:8080 - ACCOUNTS_HOST=http://localhost:5000 - HOMEPAGE_HOST=http://localhost:8000 @@ -88,10 +68,8 @@ services: - .:/offen - scriptdeps:/offen/script/node_modules command: npm start -- --port 9966 - ports: - - 9966:9966 environment: - - VAULT_HOST=http://localhost:9977 + - VAULT_HOST=http://localhost:8080/vault/ auditorium: build: @@ -102,38 +80,11 @@ services: - .:/offen - auditoriumdeps:/offen/auditorium/node_modules command: npm start -- --port 9955 - ports: - - 9955:9955 environment: - - VAULT_HOST=http://localhost:9977 - - accounts: - build: - context: '.' - dockerfile: Dockerfile.python - working_dir: /offen/accounts - volumes: - - .:/offen - - ./bootstrap.yml:/offen/accounts/bootstrap.yml - - accountdeps:/root/.local - command: flask run --host 0.0.0.0 - ports: - - 5000:5000 - links: - - accounts_database - environment: - CONFIG_CLASS: accounts.config.LocalConfig - FLASK_APP: accounts:app - FLASK_ENV: development - MYSQL_CONNECTION_STRING: mysql+pymysql://root:develop@accounts_database:3306/mysql - CORS_ORIGIN: http://localhost:9977 - SERVER_HOST: http://server:8080 - SESSION_SECRET: vndJRFJTiyjfgtTF + - VAULT_HOST=http://localhost:8080/vault/ volumes: - kmsdeps: serverdeps: scriptdeps: auditoriumdeps: vaultdeps: - accountdeps: From 8beabe7f6bd2a787e7e9662b61c4b65bcefa9d63 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sat, 31 Aug 2019 11:39:50 +0200 Subject: [PATCH 02/20] remove uneeded options, proxy homepage behind nginx too --- docker-compose.yml | 14 ++++++++++++++ homepage/docker-compose.yml | 19 ------------------- 2 files changed, 14 insertions(+), 19 deletions(-) delete mode 100644 homepage/docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml index eb351d1..a8864b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: - ./styles:/code/styles ports: - 8080:80 + - 8081:81 depends_on: - server - auditorium @@ -83,8 +84,21 @@ services: environment: - VAULT_HOST=http://localhost:8080/vault/ + homepage: + build: + context: '.' + dockerfile: ./Dockerfile.python + working_dir: /offen/homepage + volumes: + - .:/offen + - homepagedeps:/root/.local + command: make devserver + ports: + - 8000:8000 + volumes: serverdeps: scriptdeps: auditoriumdeps: vaultdeps: + homepagedeps: diff --git a/homepage/docker-compose.yml b/homepage/docker-compose.yml deleted file mode 100644 index e105b1a..0000000 --- a/homepage/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3' - -services: - homepage: - build: - context: '.' - dockerfile: ./../Dockerfile.python - working_dir: /offen/homepage - volumes: - - ..:/offen - - homepagedeps:/root/.local - command: make devserver - ports: - - 8000:8000 - environment: - DEBUG: '1' - -volumes: - homepagedeps: From 8299d745320e4ac5273415f515bb19eb25205393 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sat, 31 Aug 2019 14:35:22 +0200 Subject: [PATCH 03/20] define models, use derived keys --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index a8864b9..4234344 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ services: - 8080:80 - 8081:81 depends_on: + - homepage - server - auditorium - vault @@ -34,6 +35,7 @@ services: DEVELOPMENT: '1' COOKIE_EXCHANGE_SECRET: Wsttdo4Z3mXV5sTc EVENT_RETENTION_PERIOD: 4464h + ACCOUNT_USER_EMAIL_SALT: UwBkP24HCUYqy0Eq command: refresh run links: - server_database From 45c9b104f5da83ea8ed95c8d0d2f27224bd2af2d Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 2 Sep 2019 10:28:29 +0200 Subject: [PATCH 04/20] remove KMS --- .circleci/config.yml | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1928fab..e25a036 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,6 @@ version: 2 production_env: &production_env environment: - SERVER_HOST=https://server-alpha.offen.dev - - KMS_HOST=https://kms-alpha.offen.dev - SCRIPT_HOST=https://script-alpha.offen.dev - AUDITORIUM_HOST=https://auditorium-alpha.offen.dev - VAULT_HOST=https://vault-alpha.offen.dev @@ -15,7 +14,6 @@ production_env: &production_env deploy_preconditions: &deploy_preconditions requires: - server - - kms - vault - script - auditorium @@ -32,31 +30,6 @@ build_preconditions: &build_preconditions ignore: gh-pages jobs: - kms: - docker: - - image: circleci/golang:1.12 - environment: - - PORT=8081 - working_directory: ~/offen/kms - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-kms-{{ checksum "go.mod" }} - - run: - name: Download modules - command: go mod download - - save_cache: - paths: - - /go/pkg/mod - key: offen-kms-{{ checksum "go.mod" }} - - run: - name: Generate one-off key file - command: make bootstrap - - run: - name: Run tests - command: make test-ci - server: docker: - image: circleci/golang:1.12 @@ -297,12 +270,6 @@ jobs: name: Manually clear go cache command: sudo rm -rf /go/pkg/mod - - restore_cache: - key: offen-kms-{{ checksum "kms/go.mod" }} - - run: - name: Build kms service - working_directory: ~/offen/kms - command: make build - run: name: Manually clear go cache command: sudo rm -rf /go/pkg/mod @@ -321,8 +288,6 @@ jobs: command: | echo "Deploying server ..." $(npm bin)/sls deploy --config server/serverless.yml - echo "Deploying kms ..." - $(npm bin)/sls deploy --config kms/serverless.yml deploy_node: docker: @@ -437,8 +402,6 @@ workflows: jobs: - server: <<: *build_preconditions - - kms: - <<: *build_preconditions - vault: <<: *build_preconditions - script: From c9e40d8df5a559967194f67b9027185afe11ab58 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Tue, 3 Sep 2019 19:46:38 +0200 Subject: [PATCH 05/20] remove accounts app --- .circleci/config.yml | 73 --------- accounts/.gitignore | 8 - accounts/Makefile | 13 -- accounts/README.md | 37 ----- accounts/accounts/__init__.py | 25 --- accounts/accounts/api.py | 134 ---------------- accounts/accounts/config.py | 73 --------- accounts/accounts/models.py | 49 ------ accounts/accounts/templates/index.html | 17 -- accounts/accounts/views.py | 122 --------------- accounts/bootstrap.yml | 0 accounts/lambdas/__init__.py | 0 accounts/lambdas/authorizer.py | 80 ---------- accounts/lambdas/create_keys.py | 21 --- accounts/lambdas/rotate_keys.py | 123 --------------- accounts/requirements-dev.txt | 3 - accounts/requirements.txt | 12 -- accounts/scripts/__init__.py | 0 accounts/scripts/bootstrap.py | 41 ----- accounts/scripts/hash.py | 20 --- accounts/serverless.yml | 114 -------------- accounts/tests/__init__.py | 0 accounts/tests/test_api.py | 206 ------------------------- docker-compose.yml | 2 - package.json | 5 +- 25 files changed, 1 insertion(+), 1177 deletions(-) delete mode 100644 accounts/.gitignore delete mode 100644 accounts/Makefile delete mode 100644 accounts/README.md delete mode 100644 accounts/accounts/__init__.py delete mode 100644 accounts/accounts/api.py delete mode 100644 accounts/accounts/config.py delete mode 100644 accounts/accounts/models.py delete mode 100644 accounts/accounts/templates/index.html delete mode 100644 accounts/accounts/views.py delete mode 100755 accounts/bootstrap.yml delete mode 100644 accounts/lambdas/__init__.py delete mode 100644 accounts/lambdas/authorizer.py delete mode 100644 accounts/lambdas/create_keys.py delete mode 100644 accounts/lambdas/rotate_keys.py delete mode 100644 accounts/requirements-dev.txt delete mode 100644 accounts/requirements.txt delete mode 100644 accounts/scripts/__init__.py delete mode 100755 accounts/scripts/bootstrap.py delete mode 100644 accounts/scripts/hash.py delete mode 100644 accounts/serverless.yml delete mode 100644 accounts/tests/__init__.py delete mode 100644 accounts/tests/test_api.py diff --git a/.circleci/config.yml b/.circleci/config.yml index e25a036..56ce86d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,6 @@ production_env: &production_env - SCRIPT_HOST=https://script-alpha.offen.dev - AUDITORIUM_HOST=https://auditorium-alpha.offen.dev - VAULT_HOST=https://vault-alpha.offen.dev - - ACCOUNTS_HOST=https://accounts-alpha.offen.dev - HOMEPAGE_HOST=https://www.offen.dev - NODE_ENV=production - SECRET_ID_SERVER_CONNECTION_STRING=alpha/server/postgresConnectionString @@ -19,7 +18,6 @@ deploy_preconditions: &deploy_preconditions - auditorium - packages - shared - - accounts filters: branches: only: /^master$/ @@ -173,75 +171,6 @@ jobs: name: Run tests command: npm test - accounts: - docker: - - image: circleci/python:3.6 - environment: - CONFIG_CLASS: accounts.config.LocalConfig - MYSQL_CONNECTION_STRING: mysql://root:circle@127.0.0.1:3306/circle - - image: circleci/mysql:5.7 - environment: - - MYSQL_ROOT_PASSWORD=circle - - MYSQL_DATABASE=circle - - MYSQL_HOST=127.0.0.1 - working_directory: ~/offen/accounts - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-accounts-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} - - run: - name: Install dependencies - command: | - python3 -m venv venv - . venv/bin/activate - pip install -r requirements.txt - pip install -r requirements-dev.txt - - save_cache: - paths: - - ~/offen/accounts/venv - key: offen-accounts-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} - - run: - name: Waiting for MySQL to be ready - command: | - for i in `seq 1 10`; - do - nc -z localhost 3306 && echo Success && exit 0 - echo -n . - sleep 1 - done - echo Failed waiting for MySQL && exit 1 - - run: - name: Run tests - command: | - . venv/bin/activate - cp ~/offen/bootstrap.yml . - make test-ci - - deploy_python: - docker: - - image: circleci/python:3.6-node - <<: *production_env - working_directory: ~/offen - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-deploy-{{ checksum "package.json" }} - - run: - name: Install dependencies - command: npm install - - save_cache: - paths: - - ~/offen/packages/node_modules - key: offen-packages-{{ checksum "package.json" }} - - run: - name: Deploy - working_directory: ~/offen/accounts - command: | - echo "Deploying accounts ..." - $(npm bin)/sls deploy - deploy_golang: docker: - image: circleci/golang:1.12-node @@ -412,8 +341,6 @@ workflows: <<: *build_preconditions - shared: <<: *build_preconditions - - accounts: - <<: *build_preconditions - deploy_golang: <<: *deploy_preconditions - deploy_node: diff --git a/accounts/.gitignore b/accounts/.gitignore deleted file mode 100644 index f5bd592..0000000 --- a/accounts/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*.pem -*.dot -__pycache__ -*.log -.vscode/ -.pytest_cache -venv/ -*.pyc diff --git a/accounts/Makefile b/accounts/Makefile deleted file mode 100644 index 2c9a929..0000000 --- a/accounts/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -test: - @pytest --disable-pytest-warnings - -test-ci: bootstrap - @pytest --disable-pytest-warnings - -fmt: - @black . - -bootstrap: - @python -m scripts.bootstrap - -.PHONY: test fmt bootstrap diff --git a/accounts/README.md b/accounts/README.md deleted file mode 100644 index dd2fc7d..0000000 --- a/accounts/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# accounts - -The `accounts` app is responsible for managing operator accounts and issuing authentication tokens that will identify requests made to the `server` and the `kms` service. - -The application is built using the [Flask][flask-docs] framework. - -[flask-docs]: http://flask.pocoo.org/ - ---- - -## Development - -### Commands - -#### Install dependencies - -``` -pip install --user -r requirements.txt -r requirements-dev.txt -``` - -#### Run the development server - -``` -FLASK_APP=accounts FLASK_ENV=development flask run -``` - -#### Run the tests - -``` -make test -``` - -#### Auto format code using `black` - -``` -make fmt -``` diff --git a/accounts/accounts/__init__.py b/accounts/accounts/__init__.py deleted file mode 100644 index 6c2260e..0000000 --- a/accounts/accounts/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from os import environ - -from flask import Flask -from flask_sqlalchemy import SQLAlchemy -from flask_admin import Admin -from werkzeug.utils import import_string - - -app = Flask(__name__) - -cfg = import_string(environ.get("CONFIG_CLASS"))() -app.config.from_object(cfg) - -db = SQLAlchemy(app) - -from accounts.models import Account, User -from accounts.views import AccountView, UserView -import accounts.api - -admin = Admin( - app, name="offen admin", template_mode="bootstrap3", base_template="index.html" -) - -admin.add_view(AccountView(Account, db.session)) -admin.add_view(UserView(User, db.session)) diff --git a/accounts/accounts/api.py b/accounts/accounts/api.py deleted file mode 100644 index aad2ab4..0000000 --- a/accounts/accounts/api.py +++ /dev/null @@ -1,134 +0,0 @@ -from datetime import datetime, timedelta -from functools import wraps - -from flask import jsonify, make_response, request -from flask_cors import cross_origin -from passlib.hash import bcrypt -import jwt - -from accounts import app -from accounts.models import User - -COOKIE_KEY = "auth" - - -def json_error(handler): - @wraps(handler) - def wrapped_handler(*args, **kwargs): - try: - return handler(*args, **kwargs) - except Exception as server_error: - return ( - jsonify( - { - "error": "Internal server error: {}".format(str(server_error)), - "status": 500, - } - ), - 500, - ) - - return wrapped_handler - - -class UnauthorizedError(Exception): - pass - - -@app.route("/api/login", methods=["POST"]) -@cross_origin(origins=[app.config["CORS_ORIGIN"]], supports_credentials=True) -@json_error -def post_login(): - credentials = request.get_json(force=True) - try: - match = User.query.filter_by(email=credentials["username"]).first() - if not match: - raise UnauthorizedError("bad username") - if not bcrypt.verify(credentials["password"], match.hashed_password): - raise UnauthorizedError("bad password") - except UnauthorizedError as unauthorized_error: - resp = make_response(jsonify({"error": str(unauthorized_error), "status": 401})) - resp.set_cookie(COOKIE_KEY, "", expires=0) - resp.status_code = 401 - return resp - - expiry = datetime.utcnow() + timedelta(hours=24) - encoded = jwt.encode( - { - "exp": expiry, - "priv": { - "userId": match.user_id, - "accounts": [a.account_id for a in match.accounts], - }, - }, - app.config["JWT_PRIVATE_KEY"].encode(), - algorithm="RS256", - ).decode() - - resp = make_response(jsonify({"user": match.serialize()})) - resp.set_cookie( - COOKIE_KEY, - encoded, - httponly=True, - expires=expiry, - path="/", - domain=app.config["COOKIE_DOMAIN"], - samesite="strict", - ) - return resp - - -@app.route("/api/login", methods=["GET"]) -@cross_origin(origins=[app.config["CORS_ORIGIN"]], supports_credentials=True) -@json_error -def get_login(): - auth_cookie = request.cookies.get(COOKIE_KEY) - if not auth_cookie: - return jsonify({"error": "no auth cookie in request", "status": 401}), 401 - - public_keys = app.config["JWT_PUBLIC_KEYS"] - - token = None - token_err = None - for public_key in public_keys: - try: - token = jwt.decode(auth_cookie, public_key) - break - except Exception as decode_err: - token_err = decode_err - - if not token: - return jsonify({"error": str(token_err), "status": 401}), 401 - - try: - match = User.query.get(token["priv"]["userId"]) - except KeyError as key_err: - return ( - jsonify( - {"error": "malformed JWT claims: {}".format(key_err), "status": 401} - ), - 401, - ) - if match: - return jsonify({"user": match.serialize()}) - return jsonify({"error": "unknown user id", "status": 401}), 401 - - -@app.route("/api/logout", methods=["POST"]) -@cross_origin(origins=[app.config["CORS_ORIGIN"]], supports_credentials=True) -@json_error -def post_logout(): - resp = make_response("") - resp.set_cookie(COOKIE_KEY, "", expires=0) - resp.status_code = 204 - return resp - - -@app.route("/api/key", methods=["GET"]) -@json_error -def key(): - """ - This route is not supposed to be called by client-side applications, so - no CORS configuration is added - """ - return jsonify({"keys": app.config["JWT_PUBLIC_KEYS"]}) diff --git a/accounts/accounts/config.py b/accounts/accounts/config.py deleted file mode 100644 index 3c80bd1..0000000 --- a/accounts/accounts/config.py +++ /dev/null @@ -1,73 +0,0 @@ -import json -from os import environ - - -class BaseConfig(object): - SQLALCHEMY_TRACK_MODIFICATIONS = False - FLASK_ADMIN_SWATCH = "flatly" - - -class LocalConfig(BaseConfig): - SECRET_KEY = environ.get("SESSION_SECRET") - SQLALCHEMY_DATABASE_URI = environ.get("MYSQL_CONNECTION_STRING") - CORS_ORIGIN = environ.get("CORS_ORIGIN", "*") - COOKIE_DOMAIN = environ.get("COOKIE_DOMAIN") - SERVER_HOST = environ.get("SERVER_HOST") - def __init__(self): - with open("private_key.pem") as f: - private_key = f.read() - with open("public_key.pem") as f: - public_key = f.read() - self.JWT_PRIVATE_KEY = private_key - self.JWT_PUBLIC_KEYS = [public_key] - - -class SecretsManagerConfig(BaseConfig): - CORS_ORIGIN = environ.get("CORS_ORIGIN", "*") - COOKIE_DOMAIN = environ.get("COOKIE_DOMAIN") - SERVER_HOST = environ.get("SERVER_HOST") - - def __init__(self): - import boto3 - - session = boto3.session.Session() - self.client = session.client( - service_name="secretsmanager", region_name=environ.get("AWS_REGION") - ) - - self.SECRET_KEY = self.get_secret("sessionSecret") - self.SQLALCHEMY_DATABASE_URI = self.get_secret("mysqlConnectionString") - - current_version = self.get_secret("jwtKeyPair") - key_pair = json.loads(current_version) - previous_version = self.get_secret("jwtKeyPair", previous=True) - previous_key_pair = ( - json.loads(previous_version) - if previous_version is not None - else {"public": None} - ) - - self.JWT_PRIVATE_KEY = key_pair["private"] - self.JWT_PUBLIC_KEYS = [ - k for k in [key_pair["public"], previous_key_pair["public"]] if k - ] - - def get_secret(self, secret_name, previous=False): - import base64 - from botocore.exceptions import ClientError - - try: - ssm_response = self.client.get_secret_value( - SecretId="{}/accounts/{}".format(environ.get("STAGE"), secret_name), - VersionStage=("AWSPREVIOUS" if previous else "AWSCURRENT"), - ) - except ClientError as e: - if e.response["Error"]["Code"] == "ResourceNotFoundException" and previous: - # A secret might not have a previous version yet. It is left - # up to the caller to handle the None return in this case - return None - raise e - - if "SecretString" in ssm_response: - return ssm_response["SecretString"] - return base64.b64decode(ssm_response["SecretBinary"]) diff --git a/accounts/accounts/models.py b/accounts/accounts/models.py deleted file mode 100644 index 6068da1..0000000 --- a/accounts/accounts/models.py +++ /dev/null @@ -1,49 +0,0 @@ -from uuid import uuid4 - -from accounts import db - - -def generate_key(): - return str(uuid4()) - - -class Account(db.Model): - __tablename__ = "accounts" - account_id = db.Column(db.String(36), primary_key=True, default=generate_key) - name = db.Column(db.Text, nullable=False) - users = db.relationship("AccountUserAssociation", back_populates="account", cascade="delete") - - def __repr__(self): - return self.name - - -class User(db.Model): - __tablename__ = "users" - user_id = db.Column(db.String(36), primary_key=True, default=generate_key) - email = db.Column(db.String(128), nullable=False, unique=True) - hashed_password = db.Column(db.Text, nullable=False) - accounts = db.relationship( - "AccountUserAssociation", back_populates="user", lazy="joined" - ) - - def serialize(self): - associated_accounts = [a.account_id for a in self.accounts] - records = [ - {"name": a.name, "accountId": a.account_id} - for a in Account.query.filter(Account.account_id.in_(associated_accounts)) - ] - return {"userId": self.user_id, "email": self.email, "accounts": records} - - -class AccountUserAssociation(db.Model): - __tablename__ = "account_to_user" - - id = db.Column(db.Integer, primary_key=True) - - user_id = db.Column(db.String(36), db.ForeignKey("users.user_id"), nullable=False) - account_id = db.Column( - db.String(36), db.ForeignKey("accounts.account_id"), nullable=False - ) - - user = db.relationship("User", back_populates="accounts") - account = db.relationship("Account", back_populates="users") diff --git a/accounts/accounts/templates/index.html b/accounts/accounts/templates/index.html deleted file mode 100644 index 7a14a67..0000000 --- a/accounts/accounts/templates/index.html +++ /dev/null @@ -1,17 +0,0 @@ -{% extends 'admin/base.html' %} - -{% block head_css %} - - - - {% if admin_view.extra_css %} - {% for css_url in admin_view.extra_css %} - - {% endfor %} - {% endif %} - -{% endblock %} diff --git a/accounts/accounts/views.py b/accounts/accounts/views.py deleted file mode 100644 index e5921a3..0000000 --- a/accounts/accounts/views.py +++ /dev/null @@ -1,122 +0,0 @@ -from datetime import datetime, timedelta - -import requests -from flask_admin.contrib.sqla import ModelView -from wtforms import PasswordField, StringField, Form -from wtforms.validators import InputRequired, EqualTo -from passlib.hash import bcrypt -import jwt - -from accounts import db, app -from accounts.models import AccountUserAssociation - - -class RemoteServerException(Exception): - status = 0 - - def __str__(self): - return "Status {}: {}".format( - self.status, super(RemoteServerException, self).__str__() - ) - - -def _call_remote_server(account_id, method): - # expires in 30 seconds as this will mean the HTTP request would have - # timed out anyways - expiry = datetime.utcnow() + timedelta(seconds=30) - encoded = jwt.encode( - {"ok": True, "exp": expiry, "priv": {"rpc": "1"}}, - app.config["JWT_PRIVATE_KEY"].encode(), - algorithm="RS256", - ).decode("utf-8") - - do_request = None - if method == "POST": - do_request = requests.post - elif method == "DELETE": - do_request = requests.delete - - if not do_request: - raise Exception("Received unsupported method {}, cannot continue.".format(method)) - - r = do_request( - "{}/accounts".format(app.config["SERVER_HOST"]), - json={"accountId": account_id}, - headers={"X-RPC-Authentication": encoded}, - ) - - if r.status_code > 299: - err = r.json() - remote_err = RemoteServerException(err["error"]) - remote_err.status = err["status"] - raise remote_err - - -def create_remote_account(account_id): - return _call_remote_server(account_id, "POST") - - -def retire_remote_account(account_id): - return _call_remote_server(account_id, "DELETE") - - -class AccountForm(Form): - name = StringField( - "Account Name", - validators=[InputRequired()], - description="This is the account name visible to users", - ) - - -class AccountView(ModelView): - form = AccountForm - column_display_all_relations = True - column_list = ("name", "account_id") - - def after_model_change(self, form, model, is_created): - if is_created: - try: - create_remote_account(model.account_id) - except RemoteServerException as server_error: - db.session.delete(model) - db.session.commit() - raise server_error - - def after_model_delete(self, model): - retire_remote_account(model.account_id) - - -class UserView(ModelView): - inline_models = [(AccountUserAssociation, dict(form_columns=["id", "account"]))] - column_auto_select_related = True - column_display_all_relations = True - column_list = ("email", "user_id") - form_columns = ("email", "accounts") - form_create_rules = ("email", "password", "confirm", "accounts") - form_edit_rules = ("email", "password", "confirm", "accounts") - - def on_model_change(self, form, model, is_created): - if form.password.data: - model.hashed_password = bcrypt.hash(form.password.data) - - def get_create_form(self): - form = super(UserView, self).get_create_form() - form.password = PasswordField( - "Password", - validators=[ - InputRequired(), - EqualTo("confirm", message="Passwords must match"), - ], - ) - form.confirm = PasswordField("Repeat Password", validators=[InputRequired()]) - return form - - def get_edit_form(self): - form = super(UserView, self).get_edit_form() - form.password = PasswordField( - "Password", - description="When left blank, the password will remain unchanged on update", - validators=[EqualTo("confirm", message="Passwords must match")], - ) - form.confirm = PasswordField("Repeat Password", validators=[]) - return form diff --git a/accounts/bootstrap.yml b/accounts/bootstrap.yml deleted file mode 100755 index e69de29..0000000 diff --git a/accounts/lambdas/__init__.py b/accounts/lambdas/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/accounts/lambdas/authorizer.py b/accounts/lambdas/authorizer.py deleted file mode 100644 index ddb510d..0000000 --- a/accounts/lambdas/authorizer.py +++ /dev/null @@ -1,80 +0,0 @@ -import base64 -from os import environ - -import boto3 -from botocore.exceptions import ClientError -from passlib.hash import bcrypt - - -session = boto3.session.Session() -boto_client = session.client( - service_name="secretsmanager", region_name=environ.get("AWS_REGION") -) - - -def get_secret(boto_client, secret_name): - ssm_response = boto_client.get_secret_value( - SecretId="{}/accounts/{}".format(environ.get("STAGE"), secret_name) - ) - if "SecretString" in ssm_response: - return ssm_response["SecretString"] - return base64.b64decode(ssm_response["SecretBinary"]) - - -basic_auth_user = get_secret(boto_client, "basicAuthUser") -hashed_basic_auth_password = get_secret(boto_client, "hashedBasicAuthPassword") - - -def build_api_arn(method_arn): - arn_chunks = method_arn.split(":") - aws_region = arn_chunks[3] - aws_account_id = arn_chunks[4] - - gateway_arn_chunks = arn_chunks[5].split("/") - rest_api_id = gateway_arn_chunks[0] - stage = gateway_arn_chunks[1] - - return "arn:aws:execute-api:{}:{}:{}/{}/*/*".format( - aws_region, aws_account_id, rest_api_id, stage - ) - - -def build_response(api_arn, allow): - effect = "Deny" - if allow: - effect = "Allow" - - return { - "principalId": "offen", - "policyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": ["execute-api:Invoke"], - "Effect": effect, - "Resource": [api_arn], - } - ], - }, - } - - -def handler(event, context): - api_arn = build_api_arn(event["methodArn"]) - - encoded_auth = event["authorizationToken"].lstrip("Basic ") - auth_string = base64.standard_b64decode(encoded_auth).decode() - if not auth_string: - return build_response(api_arn, False) - - credentials = auth_string.split(":") - user = credentials[0] - password = credentials[1] - - if user != basic_auth_user: - return build_response(api_arn, False) - - if not bcrypt.verify(password, hashed_basic_auth_password): - return build_response(api_arn, False) - - return build_response(api_arn, True) diff --git a/accounts/lambdas/create_keys.py b/accounts/lambdas/create_keys.py deleted file mode 100644 index 3ba03f0..0000000 --- a/accounts/lambdas/create_keys.py +++ /dev/null @@ -1,21 +0,0 @@ -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.backends import default_backend - - -def create_key_pair(**kwargs): - key = rsa.generate_private_key( - backend=default_backend(), public_exponent=65537, **kwargs - ) - - public_key = key.public_key().public_bytes( - serialization.Encoding.PEM, serialization.PublicFormat.PKCS1 - ) - - pem = key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ) - - return {"private": pem.decode(), "public": public_key.decode()} diff --git a/accounts/lambdas/rotate_keys.py b/accounts/lambdas/rotate_keys.py deleted file mode 100644 index ce8a10e..0000000 --- a/accounts/lambdas/rotate_keys.py +++ /dev/null @@ -1,123 +0,0 @@ -import io -import boto3 -import logging -import json -from os import environ - -from lambdas.create_keys import create_key_pair - -logger = logging.getLogger() -logger.setLevel(logging.INFO) - - -def handler(event, context): - arn = event["SecretId"] - token = event["ClientRequestToken"] - step = event["Step"] - - session = boto3.session.Session() - service_client = session.client( - service_name="secretsmanager", region_name=environ.get("AWS_REGION") - ) - - # Make sure the version is staged correctly - metadata = service_client.describe_secret(SecretId=arn) - if not metadata["RotationEnabled"]: - logger.error("Secret %s is not enabled for rotation" % arn) - raise ValueError("Secret %s is not enabled for rotation" % arn) - versions = metadata["VersionIdsToStages"] - if token not in versions: - logger.error( - "Secret version %s has no stage for rotation of secret %s.", token, arn - ) - raise ValueError( - "Secret version %s has no stage for rotation of secret %s." % (token, arn) - ) - if "AWSCURRENT" in versions[token]: - logger.info( - "Secret version %s already set as AWSCURRENT for secret %s.", token, arn - ) - return - elif "AWSPENDING" not in versions[token]: - logger.error( - "Secret version %s not set as AWSPENDING for rotation of secret %s.", - token, - arn, - ) - raise ValueError( - "Secret version %s not set as AWSPENDING for rotation of secret %s." - % (token, arn) - ) - - if step == "createSecret": - create_secret(service_client, arn, token) - - elif step == "setSecret": - set_secret(service_client, arn, token) - - elif step == "testSecret": - test_secret(service_client, arn, token) - - elif step == "finishSecret": - finish_secret(service_client, arn, token) - - else: - raise ValueError("Invalid step parameter") - - -def create_secret(service_client, arn, token): - service_client.get_secret_value(SecretId=arn, VersionStage="AWSCURRENT") - try: - service_client.get_secret_value( - SecretId=arn, VersionId=token, VersionStage="AWSPENDING" - ) - logger.info("createSecret: Successfully retrieved secret for %s." % arn) - except service_client.exceptions.ResourceNotFoundException: - secret = create_key_pair(key_size=2048) - service_client.put_secret_value( - SecretId=arn, - ClientRequestToken=token, - SecretString=json.dumps(secret).encode().decode("unicode_escape"), - VersionStages=["AWSPENDING"], - ) - logger.info( - "createSecret: Successfully put secret for ARN %s and version %s." - % (arn, token) - ) - - -def set_secret(service_client, arn, token): - pass - - -def test_secret(service_client, arn, token): - pass - - -def finish_secret(service_client, arn, token): - metadata = service_client.describe_secret(SecretId=arn) - current_version = None - for version in metadata["VersionIdsToStages"]: - if "AWSCURRENT" in metadata["VersionIdsToStages"][version]: - if version == token: - # The correct version is already marked as current, return - logger.info( - "finishSecret: Version %s already marked as AWSCURRENT for %s", - version, - arn, - ) - return - current_version = version - break - - # Finalize by staging the secret version current - service_client.update_secret_version_stage( - SecretId=arn, - VersionStage="AWSCURRENT", - MoveToVersionId=token, - RemoveFromVersionId=current_version, - ) - logger.info( - "finishSecret: Successfully set AWSCURRENT stage to version %s for secret %s." - % (token, arn) - ) diff --git a/accounts/requirements-dev.txt b/accounts/requirements-dev.txt deleted file mode 100644 index bcfacb5..0000000 --- a/accounts/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest -black -pyyaml diff --git a/accounts/requirements.txt b/accounts/requirements.txt deleted file mode 100644 index ef8b1a7..0000000 --- a/accounts/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Flask==1.0.2 -Flask-Admin==1.5.3 -Flask-Cors==3.0.8 -Flask-SQLAlchemy==2.4.0 -werkzeug==0.15.4 -pyjwt[crypto]==1.7.1 -passlib==1.7.1 -bcrypt==3.1.7 -PyMySQL==0.9.3 -mysqlclient==1.4.2.post1 -requests==2.22.0 -cryptography==2.7 diff --git a/accounts/scripts/__init__.py b/accounts/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/accounts/scripts/bootstrap.py b/accounts/scripts/bootstrap.py deleted file mode 100755 index af31af1..0000000 --- a/accounts/scripts/bootstrap.py +++ /dev/null @@ -1,41 +0,0 @@ -import yaml -from passlib.hash import bcrypt - -from lambdas.create_keys import create_key_pair - -if __name__ == "__main__": - keypair = create_key_pair(key_size=2048) - with open("public_key.pem", "w") as f: - f.write(keypair["public"]) - - with open("private_key.pem", "w") as f: - f.write(keypair["private"]) - - from accounts import db - from accounts.models import Account, User, AccountUserAssociation - - db.drop_all() - db.create_all() - - with open("./bootstrap.yml", "r") as stream: - data = yaml.safe_load(stream) - - for account in data["accounts"]: - record = Account( - name=account["name"], - account_id=account["id"], - ) - db.session.add(record) - - for user in data["users"]: - record = User( - email=user["email"], - hashed_password=bcrypt.hash(user["password"]), - ) - for account_id in user["accounts"]: - record.accounts.append(AccountUserAssociation(account_id=account_id)) - db.session.add(record) - - db.session.commit() - - print("Successfully bootstrapped accounts database and created key pair") diff --git a/accounts/scripts/hash.py b/accounts/scripts/hash.py deleted file mode 100644 index 23e66ab..0000000 --- a/accounts/scripts/hash.py +++ /dev/null @@ -1,20 +0,0 @@ -import base64 -import argparse - -from passlib.hash import bcrypt - -parser = argparse.ArgumentParser() -parser.add_argument("--password", type=str, help="The password to hash", required=True) -parser.add_argument( - "--plain", - help="Do not encode the result as base64", - default=False, - action="store_true", -) - -if __name__ == "__main__": - args = parser.parse_args() - out = bcrypt.hash(args.password) - if not args.plain: - out = base64.standard_b64encode(out.encode()).decode() - print(out) diff --git a/accounts/serverless.yml b/accounts/serverless.yml deleted file mode 100644 index 017bc31..0000000 --- a/accounts/serverless.yml +++ /dev/null @@ -1,114 +0,0 @@ -service: - name: accounts - awsKmsKeyArn: ${ssm:/aws/reference/secretsmanager/${self:custom.stage}/all/kmsArn~true} - -provider: - name: aws - endpointType: regional - runtime: python3.6 - stage: alpha - region: eu-central-1 - apiName: offen-${self:provider.stage} - logs: - restApi: true - iamRoleStatements: - - Effect: 'Allow' - Action: - - secretsmanager:GetSecretValue - Resource: arn:aws:secretsmanager:eu-central-1:#{AWS::AccountId}:secret:${self:custom.stage}/* - -package: - individually: true - exclude: - - tests - -plugins: - - serverless-domain-manager - - serverless-python-requirements - - serverless-wsgi - - serverless-pseudo-parameters - -custom: - stage: ${opt:stage, self:provider.stage} - origin: - production: https://vault.offen.dev - staging: https://vault-staging.offen.dev - alpha: https://vault-alpha.offen.dev - serverHost: - production: https://server.offen.dev - staging: https://server-staging.offen.dev - alpha: https://server-alpha.offen.dev - domain: - production: accounts.offen.dev - staging: accounts-staging.offen.dev - alpha: accounts-alpha.offen.dev - cookieDomain: - production: .offen.dev - staging: .offen.dev - alpha: .offen.dev - customDomain: - basePath: '' - certificateName: '*.offen.dev' - domainName: ${self:custom.domain.${self:custom.stage}} - stage: ${self:custom.stage} - endpointType: regional - createRoute53Record: false - wsgi: - app: accounts.app - packRequirements: false - pythonRequirements: - slim: true - dockerizePip: non-linux - fileName: requirements.txt - -functions: - authorizer: - handler: lambdas.authorizer.handler - environment: - STAGE: ${self:custom.stage} - rotateKeys: - handler: lambdas.rotate_keys.handler - environment: - STAGE: ${self:custom.stage} - app: - handler: wsgi_handler.handler - timeout: 30 - events: - - http: - path: /admin/ - method: any - authorizer: - name: authorizer - resultTtlInSeconds: 0 - identitySource: method.request.header.Authorization - - http: - path: /admin/{proxy+} - method: any - authorizer: - name: authorizer - resultTtlInSeconds: 0 - identitySource: method.request.header.Authorization - - http: - path: '/' - method: any - - http: - path: '/{proxy+}' - method: any - environment: - CONFIG_CLASS: accounts.config.SecretsManagerConfig - STAGE: ${self:custom.stage} - CORS_ORIGIN: ${self:custom.origin.${self:custom.stage}} - COOKIE_DOMAIN: ${self:custom.cookieDomain.${self:custom.stage}} - SERVER_HOST: ${self:custom.serverHost.${self:custom.stage}} - -resources: - Resources: - GatewayResponse: - Type: 'AWS::ApiGateway::GatewayResponse' - Properties: - ResponseParameters: - gatewayresponse.header.WWW-Authenticate: "'Basic'" - ResponseType: UNAUTHORIZED - RestApiId: - Ref: 'ApiGatewayRestApi' - StatusCode: '401' diff --git a/accounts/tests/__init__.py b/accounts/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/accounts/tests/test_api.py b/accounts/tests/test_api.py deleted file mode 100644 index a9c8512..0000000 --- a/accounts/tests/test_api.py +++ /dev/null @@ -1,206 +0,0 @@ -import unittest -import json -import base64 -from time import time -from datetime import datetime, timedelta -from os import environ - -import jwt - -from accounts import app - - -FOREIGN_PRIVATE_KEY = """ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCwAPFiTSLKlVvG -N97TIyDWIxPp4Ji8hAmtlMn0gdGclC2DGKA2v7orXdNkngFon0PPe08acKI5NL9P -nkVSrjWxrn8H7LeNQadwPxjYVmri4SLhBJUcAe+SoqrIZtrci+2y64mLPrl6wxBj -ZKDl8o1Qm8iZSMgJ+wRG2FrItZUBWLZ79KSB2lQkO5OWorPX3T0SPxQXqq9hc4xN -6I+qtfmv5jZTJOviMCehOs48ZlObgr/W+Kak4q/jrrqXvG3XQqVVTN/z95+2XuN4 -Btj7fv24PIRE/BddDAzC/yzISYb9QqLChaxx1fqY+aSA6ou2wh1PjUiyXNnAmP2i -6UWwikILAgMBAAECggEBAJuYmc1/x+w00qeQKQubmKH27NnsVtsCF9Q/H7NrOTYl -wX6OPMVqBlnkXsgq76/gbQB2UN5dCO1t9lua3kpT/OASFfeZjEPy8OXIwlwvOdtN -kZpAhNn31CZcbIMyevZTNlbg5/4T+8HNxSU5hw0Cu2+x6UuqDj7UjVlcWBXsgchn -f8kguLHr6Q7rndC10Vv5a4Rz9fzuS2K4jEnhlJjgD22XB2SCH5kLrAikH10AW761 -5g7HSiMxKSUyXc51PX3n/FkxjzT0Vm1ENeZou263VEQhke49IWLIcbLD7ShOyNaI -TuYPAyRY4o70/d/YTydRCEp/H8stB6UaVK9hlzzfoMECgYEA1e9UgW4vBueSoZv3 -llc7dXlAnk6nJeCaujjPBAd0Sc3XcpMik1kb8oDgI4fwNxTYqlHu3Wp7ZLR14C4G -rlry+4rRUdxnWNcKtyOtA6km0b33V3ja4GsLViENBSQZDUe7EljER2VSRynMTog0 -lfmUr+ORzWDpanEO+Ke25zhU2DsCgYEA0pxM2UjmmAepSWBAcXABjIFE09MxXVTS -NwRhdYjHJsKmGnPD8DEDJbRSHNAEN2mTD2kJW5pFThKVWtQ8WpjSXuRSkS7HzXrU -zMNZnzTDdTZl6nnui3RJtIYntSXR7ommC6ldY7nlnHnzkIEcDLwN6E/JNOB5gtTE -L4ztUpKncHECgYBO3qHX6agasorjW52mZlh8UYxaEIMcurYwSzs+sATWJLX1/npz -uhlMiOiZEMelduD9waD/Lf95u/HtCOrbopoL1DyhIlFTdkv0AooJXHX8Qz2JmPuQ -WsZeJWcoawt1UumLtP//lkIEDEvO8/X3CIEhaxNYlQ7Yd//d+e67RZA5+wKBgD6f -qR4m1iI4jPa7fw377wn3Wh7eOlx1Hziqvcv0CruUv004RPfDqxrn/k6A7/AGHWtE -oTqyqY7oaa6jUvrhXBRJMd/nmBOaRXJJV/nF96R/s1hAP1UKE+xww5fSkhSqq0vm -ZVWE7ihT/r9mFJAYzs3YA40MfjUPzPISpnKaFt2RAoGBANCtswMqztcuPDF5rL3d -rqB6jwFrXKvwrx4HxOmF/MgGPyp6MWLBEnpZDvLJo9uSafq6Q6IwOQMWWF5GO7JO -4EG9ldVugR/CtmL3+XTHE4MGPXmqHg/q/o7rItc7g11iXJTndcUZtWGwkHwl4zBF -15NFZ2gU4rKnQ3sVAOzMoEw5 ------END PRIVATE KEY----- -""" - -def _pad_b64_string(s): - while len(s) % 4 is not 0: - s = s + "=" - return s - - -class TestKey(unittest.TestCase): - def setUp(self): - self.app = app.test_client() - - def test_get_key(self): - rv = self.app.get("/api/key") - assert rv.status.startswith("200") - data = json.loads(rv.data) - assert data["keys"] - - -class TestJWT(unittest.TestCase): - def setUp(self): - self.app = app.test_client() - - def _assert_cookie_present(self, name): - for cookie in self.app.cookie_jar: - if cookie.name == name: - return cookie.value - raise AssertionError("Cookie named {} not found".format(name)) - - def _assert_cookie_not_present(self, name): - for cookie in self.app.cookie_jar: - assert cookie.name != name - - def test_jwt_flow(self): - """ - First, try login attempts that are supposed to fail: - 1. checking login status without any prior interaction - 2. try logging in with an unknown user - 3. try logging in with a known user and bad password - """ - rv = self.app.get("/api/login") - assert rv.status.startswith("401") - self._assert_cookie_not_present("auth") - - rv = self.app.post( - "/api/login", - data=json.dumps( - {"username": "does@not.exist", "password": "somethingsomething"} - ), - ) - assert rv.status.startswith("401") - self._assert_cookie_not_present("auth") - - rv = self.app.post( - "/api/login", - data=json.dumps({"username": "develop@offen.dev", "password": "developp"}), - ) - assert rv.status.startswith("401") - self._assert_cookie_not_present("auth") - - """ - Next, perform a successful login - """ - rv = self.app.post( - "/api/login", - data=json.dumps({"username": "develop@offen.dev", "password": "develop"}), - ) - assert rv.status.startswith("200") - - """ - The response should contain information about the - user and full information (i.e. a name) about the associated accounts - """ - data = json.loads(rv.data) - assert data["user"]["userId"] is not None - data["user"]["accounts"].sort(key=lambda a: a["name"]) - self.assertListEqual( - data["user"]["accounts"], - [ - {"name": "One", "accountId": "9b63c4d8-65c0-438c-9d30-cc4b01173393"}, - {"name": "Two", "accountId": "78403940-ae4f-4aff-a395-1e90f145cf62"}, - ], - ) - - """ - The claims part of the JWT is expected to contain a valid expiry, - information about the user and the associated account ids. - """ - jwt = self._assert_cookie_present("auth") - # PyJWT strips the padding from the base64 encoded parts which Python - # cannot decode properly, so we need to add the padding ourselves - claims_part = _pad_b64_string(jwt.split(".")[1]) - claims = json.loads(base64.b64decode(claims_part)) - assert claims.get("exp") > time() - - priv = claims.get("priv") - assert priv is not None - - assert priv.get("userId") is not None - self.assertListEqual( - priv["accounts"], - [ - "9b63c4d8-65c0-438c-9d30-cc4b01173393", - "78403940-ae4f-4aff-a395-1e90f145cf62", - ], - ) - - """ - Checking the login status when re-using the cookie should yield - a successful response - """ - rv = self.app.get("/api/login") - assert rv.status.startswith("200") - jwt2 = self._assert_cookie_present("auth") - assert jwt2 == jwt - - """ - Performing a bad login attempt when sending a valid auth cookie - is expected to destroy the cookie and leave the user logged out again - """ - rv = self.app.post( - "/api/login", - data=json.dumps( - {"username": "evil@session.takeover", "password": "develop"} - ), - ) - assert rv.status.startswith("401") - self._assert_cookie_not_present("auth") - - """ - Explicitly logging out leaves the user without cookies - """ - rv = self.app.post( - "/api/login", - data=json.dumps({"username": "develop@offen.dev", "password": "develop"}), - ) - assert rv.status.startswith("200") - - rv = self.app.post("/api/logout") - assert rv.status.startswith("204") - self._assert_cookie_not_present("auth") - - def test_forged_token(self): - """ - The application needs to verify that tokens that would be theoretically - valid are not signed using an unknown key. - """ - forged_token = jwt.encode( - { - "exp": datetime.utcnow() + timedelta(hours=24), - "priv": { - "userId": "8bc8db1b-f32d-4376-a1cf-724bf6a597b8", - "accounts": [ - "9b63c4d8-65c0-438c-9d30-cc4b01173393", - "78403940-ae4f-4aff-a395-1e90f145cf62", - ], - }, - }, - FOREIGN_PRIVATE_KEY, - algorithm="RS256", - ).decode() - - self.app.set_cookie("localhost", "auth", forged_token) - rv = self.app.get("/api/login") - assert rv.status.startswith("401") diff --git a/docker-compose.yml b/docker-compose.yml index 4234344..3a9ecaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,6 @@ services: POSTGRES_CONNECTION_STRING: postgres://postgres:develop@server_database:5432/postgres?sslmode=disable KMS_ENCRYPTION_ENDPOINT: http://kms:8081/encrypt PORT: 8080 - JWT_PUBLIC_KEY: http://accounts:5000/api/key DEVELOPMENT: '1' COOKIE_EXCHANGE_SECRET: Wsttdo4Z3mXV5sTc EVENT_RETENTION_PERIOD: 4464h @@ -59,7 +58,6 @@ services: - KMS_HOST=http://localhost:8081 - SCRIPT_HOST=http://localhost:9977 - AUDITORIUM_HOST=http://localhost:8080 - - ACCOUNTS_HOST=http://localhost:5000 - HOMEPAGE_HOST=http://localhost:8000 script: diff --git a/package.json b/package.json index c0b426f..05f0297 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,9 @@ "homepage": "https://github.com/offen/offen#readme", "dependencies": { "serverless": "^1.45.0", - "serverless-apigw-binary": "^0.4.4", "serverless-domain-manager": "^2.6.13", "serverless-finch": "^2.4.2", - "serverless-pseudo-parameters": "^2.4.0", - "serverless-python-requirements": "^4.3.0", - "serverless-wsgi": "^1.7.2" + "serverless-pseudo-parameters": "^2.4.0" }, "devDependencies": {} } From 9ee20f37d076bc17ff52ba0ecf415ac4cc564f25 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 4 Sep 2019 15:22:46 +0200 Subject: [PATCH 06/20] remove alpha infrastructure configuration --- .circleci/config.yml | 105 ------------------------------------------- docker-compose.yml | 2 - package.json | 22 --------- 3 files changed, 129 deletions(-) delete mode 100644 package.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 56ce86d..8bb2678 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -171,105 +171,6 @@ jobs: name: Run tests command: npm test - deploy_golang: - docker: - - image: circleci/golang:1.12-node - <<: *production_env - working_directory: ~/offen - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-deploy-{{ checksum "package.json" }} - - run: - name: Install dependencies - command: npm install - - save_cache: - paths: - - ~/offen/packages/node_modules - key: offen-packages-{{ checksum "package.json" }} - - - restore_cache: - key: offen-server-{{ checksum "server/go.mod" }} - - run: - name: Build server service - working_directory: ~/offen/server - command: make build - - run: - name: Manually clear go cache - command: sudo rm -rf /go/pkg/mod - - - run: - name: Manually clear go cache - command: sudo rm -rf /go/pkg/mod - - - run: - name: Migrate `server` database - working_directory: ~/offen/server - command: | - sudo apt-get update && sudo apt-get install -qq -y python-pip libpython-dev - curl -O https://bootstrap.pypa.io/get-pip.py && sudo python get-pip.py - sudo pip install -q awscli --upgrade - go run cmd/migrate/main.go -conn $(aws secretsmanager get-secret-value --secret-id $SECRET_ID_SERVER_CONNECTION_STRING | jq -r '.SecretString') - - run: - name: Deploy - working_directory: ~/offen - command: | - echo "Deploying server ..." - $(npm bin)/sls deploy --config server/serverless.yml - - deploy_node: - docker: - - image: circleci/node:10 - <<: *production_env - working_directory: ~/offen - steps: - - checkout: - path: ~/offen - - restore_cache: - key: offen-deploy-{{ checksum "package.json" }} - - run: - name: Install dependencies - command: npm install - - save_cache: - paths: - - ~/offen/packages/node_modules - key: offen-packages-{{ checksum "package.json" }} - - restore_cache: - key: offen-auditorium-{{ checksum "auditorium/package.json" }} - - run: - name: Build auditorium service - working_directory: ~/offen/auditorium - command: npm run build - - - restore_cache: - key: offen-script-{{ checksum "script/package.json" }} - - run: - name: Build script service - working_directory: ~/offen/script - command: npm run build - - - restore_cache: - key: offen-vault-{{ checksum "vault/package.json" }} - - run: - name: Build vault service - working_directory: ~/offen/vault - command: npm run build - - - run: - name: Deploy - working_directory: ~/offen - command: | - echo "Deploying auditorium ..." - $(npm bin)/sls deploy --config auditorium/serverless.yml - $(npm bin)/sls client deploy --config auditorium/serverless.yml --no-confirm - echo "Deploying script ..." - $(npm bin)/sls deploy --config script/serverless.yml - $(npm bin)/sls client deploy --config script/serverless.yml --no-confirm - echo "Deploying vault ..." - $(npm bin)/sls deploy --config vault/serverless.yml - $(npm bin)/sls client deploy --config vault/serverless.yml --no-confirm - deploy_homepage: docker: - image: circleci/python:3.6-node @@ -341,11 +242,5 @@ workflows: <<: *build_preconditions - shared: <<: *build_preconditions - - deploy_golang: - <<: *deploy_preconditions - - deploy_node: - <<: *deploy_preconditions - - deploy_python: - <<: *deploy_preconditions - deploy_homepage: <<: *deploy_preconditions diff --git a/docker-compose.yml b/docker-compose.yml index 3a9ecaa..b2d4bf4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,9 +27,7 @@ services: - serverdeps:/go/pkg/mod environment: CORS_ORIGIN: http://localhost:9977 - OPTOUT_COOKIE_DOMAIN: localhost POSTGRES_CONNECTION_STRING: postgres://postgres:develop@server_database:5432/postgres?sslmode=disable - KMS_ENCRYPTION_ENDPOINT: http://kms:8081/encrypt PORT: 8080 DEVELOPMENT: '1' COOKIE_EXCHANGE_SECRET: Wsttdo4Z3mXV5sTc diff --git a/package.json b/package.json deleted file mode 100644 index 05f0297..0000000 --- a/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "offen", - "private": true, - "version": "1.0.0", - "repository": { - "type": "git", - "url": "git+https://github.com/offen/offen.git" - }, - "author": "offen", - "license": "MIT", - "bugs": { - "url": "https://github.com/offen/offen/issues" - }, - "homepage": "https://github.com/offen/offen#readme", - "dependencies": { - "serverless": "^1.45.0", - "serverless-domain-manager": "^2.6.13", - "serverless-finch": "^2.4.2", - "serverless-pseudo-parameters": "^2.4.0" - }, - "devDependencies": {} -} From eca5b954b48f25bc239076ba133351f6f328c70c Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 4 Sep 2019 16:20:35 +0200 Subject: [PATCH 07/20] try mounting auditorium in homepage --- docker-compose.yml | 4 +- homepage/content/pages/auditorium.md | 5 + homepage/pelicanconf.py | 1 + homepage/theme/templates/auditorium.html | 11 +- homepage/theme/templates/base.html | 8 +- styles/index.css | 425 ----------------------- 6 files changed, 20 insertions(+), 434 deletions(-) create mode 100644 homepage/content/pages/auditorium.md diff --git a/docker-compose.yml b/docker-compose.yml index b2d4bf4..575f8f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: - KMS_HOST=http://localhost:8081 - SCRIPT_HOST=http://localhost:9977 - AUDITORIUM_HOST=http://localhost:8080 - - HOMEPAGE_HOST=http://localhost:8000 + - HOMEPAGE_HOST=http://localhost:8081 script: build: @@ -93,6 +93,8 @@ services: command: make devserver ports: - 8000:8000 + environment: + DEBUG: 1 volumes: serverdeps: diff --git a/homepage/content/pages/auditorium.md b/homepage/content/pages/auditorium.md new file mode 100644 index 0000000..463761a --- /dev/null +++ b/homepage/content/pages/auditorium.md @@ -0,0 +1,5 @@ +Title: Auditorium | offen +description: offen is a free and open source analytics software for websites and web applications that allows respectful handling of data. +save_as: auditorium/index.html +href: /auditorium/ +template: auditorium diff --git a/homepage/pelicanconf.py b/homepage/pelicanconf.py index 9a786bf..1fd3ff4 100644 --- a/homepage/pelicanconf.py +++ b/homepage/pelicanconf.py @@ -43,3 +43,4 @@ DIRECT_TEMPLATES = [] GITHUB_ORG = 'https://github.com/offen' CONTACT_EMAIL = 'mail@offen.dev' PATREON_URL = 'https://www.patreon.com/bePatron?u=21484999' +AUDITORIUM_SCRIPT = 'http://localhost:8080/auditorium/index.js' diff --git a/homepage/theme/templates/auditorium.html b/homepage/theme/templates/auditorium.html index 1462aa3..997789b 100644 --- a/homepage/theme/templates/auditorium.html +++ b/homepage/theme/templates/auditorium.html @@ -1,9 +1,10 @@ {% extends "base.html" %} {% block content %} -
-
- {{ page.content }} -
-
+
+{% endblock %} + +{% block scripts %} + {{ super () }} + {% endblock %} diff --git a/homepage/theme/templates/base.html b/homepage/theme/templates/base.html index 4f409d8..26b3792 100644 --- a/homepage/theme/templates/base.html +++ b/homepage/theme/templates/base.html @@ -136,8 +136,10 @@ - {% assets filters="rjsmin", output="scripts/packed.js", "scripts/jquery-3.4.1.min.js", "scripts/menu.js", "scripts/fade.js" %} - - {% endassets %} + {% block scripts %} + {% assets filters="rjsmin", output="scripts/packed.js", "scripts/jquery-3.4.1.min.js", "scripts/menu.js", "scripts/fade.js" %} + + {% endassets %} + {% endblock %} diff --git a/styles/index.css b/styles/index.css index 44ceb40..e69de29 100644 --- a/styles/index.css +++ b/styles/index.css @@ -1,425 +0,0 @@ -/* Typo -------------------------------------------------------------- */ -body { -padding: 50px; -background-color: #fff; -color: #898989; -font: 18px/1.5 "Noto Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; -font-size: 18px; -line-height: 1.5; -font-weight: 400; -} - -h1, -h2, -h3, -h4 { -color: #404040; -font-size: 28px; -line-height: 1.4; -font-weight: 400; -margin: 0 0 40px 0; -} - -h3, -h4 { -font-size: 18px; -margin: 30px 0 20px 0; -} - -.section-auditorium h3 { -margin: 0 0 10px 0; -} - -h4 { -color: #898989; -margin: 30px 0 6px 0; -} - -.section-auditorium h4 { -margin: 30px 0 20px 0; -} - - -/* Links -------------------------------------------------------------- */ -a, -a:hover, -a:focus { -color: #898989; -text-decoration: none; -font-weight: 700; -} - -button { -cursor: pointer; -font: inherit; -font-size: inherit; -font-weight: 700; -} - -h1 a, -h1 a:hover, -h1 a:focus { -color: #f7bf08; -text-decoration: none; -font-weight: 400; -} - -small a, -small a:hover, -small a:focus { -color: #898989; -text-decoration: none; -font-weight: 700; -} - -sup { -line-height: 1; -} - -sup a, -sup a:hover, -sup a:focus { -color: #cfcfcf; -} - -ol { -padding-inline-start: 20px; -} - -ol li p a, -ol li p a:hover, -ol li p a:focus { -color: #cfcfcf; -} - - - - -/* Typo Elements -------------------------------------------------------------- */ -.logo { -margin: -20px 0 0 0; -} - -blockquote { -border-left: 1px solid #e5e5e5; -margin: 0; -padding: 0 0 0 20px; -font-style: italic -} - -hr { -border-top: 1px solid #d5d5d5; -height: 0; -margin: 80px 0 0 0; -} - -.footnote hr { -border: 0; -height: 0; -margin: 0; -} - -blockquote { -border-left: 1px solid #d5d5d5; -margin: 40px 0 40px 0; -} - -strong { -font-weight: 700 -} - -strong, -h1 strong, -h2 strong, -h3 strong { -color: #404040; -text-decoration: none; -font-weight: 400; -} - -h1 strong, -h2 strong, -h3 strong { -color: #898989; -} - -tbody, -h4 strong { -color: #404040; -text-decoration: none; -font-weight: 700; -} - -h4 strong { -font-size: 36px; -} - - -/* Buttons -------------------------------------------------------------- */ -.btn { --webkit-border-radius: 0; --moz-border-radius: 0; -border-radius: 0px; -padding: 6px; -text-align: center; -text-decoration: none; -} - -.btn-color-grey, -.btn-color-grey:visited, -.btn-color-grey:link { -background: #fff; -color: #898989; -border: solid #898989 5px; -} - -.btn-color-orange, -.btn-color-orange:visited, -.btn-color-orange:link { -background: #fff; -color: #F7BF08; -border: solid #F7BF08 5px; -} - -.btn-color-grey:hover, -.btn-color-grey:active { -background: #898989; -color: #fff; -text-decoration: none; -} - -.btn-color-orange:hover, -.btn-color-orange:active { -background: #F7BF08; -color: #fff; -text-decoration: none; -} - -.btn { -margin: 0 0 15px 0; -} - -.button-wrapper { -display: flex; -flex-flow: column wrap; -justify-content: space-around; -} - -.navigation-wrapper { -margin: 60px 0 0 0; -} - -@media (min-width: 480px) { -.button-wrapper { -align-items: center; -flex-flow: row wrap; --ms-flex-flow: row wrap; --webkit-flex-flow: row wrap; -} -.btn { -flex-grow: 1; -} -.btn { -margin: 0 15px 15px 0; -} -.btn:last-of-type { -margin: 0 0 15px 0; -} -} - - - - -/* Custom Underline -------------------------------------------------------------- */ -em { -color: #898989; -background: linear-gradient(transparent 66%, #fde28c 66%); -font-style: normal; -} - - - - -/* Layout -------------------------------------------------------------- */ -.wrapper { -width: 860px; -margin: 0 auto -} - -header { -width: 270px; -float: left; -position: fixed; --webkit-font-smoothing: subpixel-antialiased -} - -section, -section-auditorium { -float: right; -width: 600px; -padding: 0 0 40px 0; -} - -section > p, -section-auditorium > p { -margin: 0 0 20px 0; -} - -section > p:last-child, -section-auditorium > p:last-child { -margin: 0 0 60px 0; -} - -.row { -display: flex; -justify-content: flex-start; -} - -.row h4 { -margin-right: 30px; -} - -.row h4:last-child { -margin-right: 0; -} - - - - -/* Tables -------------------------------------------------------------- */ -.table-full-width { -box-sizing: border-box -} - -td { -padding-right: 30px; -} - - - - -/* Footer -------------------------------------------------------------- */ -footer { -width: 270px; -float: left; -position: fixed; -bottom: 100px; --webkit-font-smoothing: subpixel-antialiased -} - -.footnote { -color: #d5d5d5; -font-size: 14px; -margin: 20px 0 0 0; -} - -.footnote ol { -margin-bottom: 0; -} - -.footer-list { -list-style-type: none; -padding: 0; -margin: 0; -} - -.footer-list li { -display: inline-block; -font-size: 14px; -margin-right: 0.14em; -} - - - - -/* Media Queries -------------------------------------------------------------- */ -@media screen and (max-width: 960px) { -div.wrapper { -width: auto; -margin: 0 -} -header, -section, -section-auditorium, -footer { -float: none; -position: static; -width: auto -} -header { -padding-right: 320px -} -section, -section-auditorium { -border: none; -width: auto; -padding: 0 0 0 0; -margin: 40px 0 40px 0; -} -section > p:first-child, -section > p:last-child, -section-auditorium > p:first-child, -section-auditorium > p:last-child { -margin: 0 0 40px 0; -} -} - - - - -@media screen and (max-width: 480px) { -body { -font-size: 14px; -line-height: 1.3; -padding: 15px; -word-wrap: break-word; -} -h1, -h2 { -font-size: 20px; -line-height: 1.3; -margin: 0 0 20px 0; -} -h3, -h4 { -font-size: 14px; -line-height: 1.3; -margin: 30px 0 15px 0; -} -h4 strong { -font-size: 28px; -} -header { -padding: 0 -} -.logo { -margin: 0; -} -hr { -margin: 60px 0 0 0; -} -blockquote { -margin: 15px 0 15px 0; -} -.row { -justify-content: space-between; -} -.row h4:first-child { -margin-right: 0; -} -.table-full-width { -width: 100%; -} -td { -padding-right: 0; -} -} From 907736a6e26d614c82d2efdd95855e02d150cc33 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 4 Sep 2019 17:32:28 +0200 Subject: [PATCH 08/20] use single host for server, assets and homepage --- docker-compose.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 575f8f2..ed4d640 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,12 +51,6 @@ services: - .:/offen - vaultdeps:/offen/vault/node_modules command: npm start -- --port 9977 - environment: - - SERVER_HOST=http://localhost:8080 - - KMS_HOST=http://localhost:8081 - - SCRIPT_HOST=http://localhost:9977 - - AUDITORIUM_HOST=http://localhost:8080 - - HOMEPAGE_HOST=http://localhost:8081 script: build: @@ -68,7 +62,7 @@ services: - scriptdeps:/offen/script/node_modules command: npm start -- --port 9966 environment: - - VAULT_HOST=http://localhost:8080/vault/ + - VAULT_HOST=/vault/ auditorium: build: @@ -80,7 +74,7 @@ services: - auditoriumdeps:/offen/auditorium/node_modules command: npm start -- --port 9955 environment: - - VAULT_HOST=http://localhost:8080/vault/ + - VAULT_HOST=/vault/ homepage: build: From b2fe172681da43c124fd703d7a9f2442d5028b39 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 4 Sep 2019 20:50:53 +0200 Subject: [PATCH 09/20] add dockerized deployment build --- build/docker-compose.yml | 16 +++++++++++ build/proxy/Dockerfile | 58 ++++++++++++++++++++++++++++++++++++++++ build/proxy/nginx.conf | 22 +++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 build/docker-compose.yml create mode 100644 build/proxy/Dockerfile create mode 100644 build/proxy/nginx.conf diff --git a/build/docker-compose.yml b/build/docker-compose.yml new file mode 100644 index 0000000..b45db47 --- /dev/null +++ b/build/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3' + +services: + proxy: + build: + context: './..' + dockerfile: build/proxy/Dockerfile + ports: + - 3000:80 + depends_on: + - server + + server: + build: + context: './..' + dockerfile: build/server/Dockerfile diff --git a/build/proxy/Dockerfile b/build/proxy/Dockerfile new file mode 100644 index 0000000..40dd5b2 --- /dev/null +++ b/build/proxy/Dockerfile @@ -0,0 +1,58 @@ +FROM node:10 as auditorium + +COPY ./auditorium /code/auditorium +COPY ./packages /code/packages +COPY ./styles /code/styles +COPY ./banner.txt /code/banner.txt +WORKDIR /code/auditorium +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN npm install +ENV NODE_ENV production +RUN npm run build + +FROM node:10 as script + +COPY ./script /code/script +COPY ./packages /code/packages +COPY ./banner.txt /code/banner.txt +WORKDIR /code/script +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN npm install +ENV NODE_ENV production +RUN npm run build + +FROM node:10 as vault + +COPY ./vault /code/vault +COPY ./packages /code/packages +COPY ./banner.txt /code/banner.txt +WORKDIR /code/vault +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN npm install +ENV NODE_ENV production +RUN npm run build + +FROM nikolaik/python-nodejs:python3.6-nodejs10 as homepage + +COPY ./homepage /code/homepage +COPY ./styles /code/styles + +RUN npm install svgo -g +RUN apt-get update \ + && apt-get install -y libjpeg-progs optipng \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /code/homepage +ENV PATH /root/.local/bin:$PATH +RUN pip install --user -r requirements.txt +RUN make publish + +FROM nginx:1.17-alpine + +COPY --from=homepage /code/homepage/output /www/data +COPY --from=auditorium /code/auditorium/dist /www/data/auditorium +COPY --from=script /code/script/dist /www/data/script +COPY --from=vault /code/vault/dist /www/data/vault +COPY ./build/proxy/nginx.conf /etc/nginx/nginx.conf + +EXPOSE 80 diff --git a/build/proxy/nginx.conf b/build/proxy/nginx.conf new file mode 100644 index 0000000..ae70f49 --- /dev/null +++ b/build/proxy/nginx.conf @@ -0,0 +1,22 @@ +events {} + +http { + include /etc/nginx/mime.types; + + upstream server { + server server:3000; + } + + server { + listen 80; + autoindex on; + root /www/data; + + location /api/ { + proxy_pass http://server; + proxy_redirect off; + rewrite ^/api(.*)$ $1 break; + } + } +} + From 460e2a77acde0fafb334e26668fd1cbf57d7d637 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 5 Sep 2019 11:18:42 +0200 Subject: [PATCH 10/20] normalize local urls throughout --- .circleci/config.yml | 15 -- Makefile | 6 +- build/docker-compose.yml | 20 +- build/proxy/Dockerfile | 2 +- build/proxy/nginx.conf | 12 +- docker-compose.yml | 6 - homepage/content/pages/auditorium.md | 5 - homepage/publishconf.py | 4 +- homepage/theme/templates/auditorium.html | 10 - homepage/theme/templates/base.html | 4 +- shared/Makefile | 2 - shared/README.md | 15 -- shared/http/errors.go | 24 -- shared/http/errors_test.go | 19 -- shared/http/jwt.go | 174 -------------- shared/http/jwt_test.go | 279 ----------------------- shared/http/middleware.go | 64 ------ shared/http/middleware_test.go | 122 ---------- styles/index.css | 0 19 files changed, 35 insertions(+), 748 deletions(-) delete mode 100644 homepage/content/pages/auditorium.md delete mode 100644 homepage/theme/templates/auditorium.html delete mode 100644 shared/Makefile delete mode 100644 shared/README.md delete mode 100644 shared/http/errors.go delete mode 100644 shared/http/errors_test.go delete mode 100644 shared/http/jwt.go delete mode 100644 shared/http/jwt_test.go delete mode 100644 shared/http/middleware.go delete mode 100644 shared/http/middleware_test.go delete mode 100644 styles/index.css diff --git a/.circleci/config.yml b/.circleci/config.yml index 8bb2678..bc4960e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,6 @@ deploy_preconditions: &deploy_preconditions - script - auditorium - packages - - shared filters: branches: only: /^master$/ @@ -68,20 +67,6 @@ jobs: cp ~/offen/bootstrap.yml . make test-ci - shared: - docker: - - image: circleci/golang:1.12 - working_directory: ~/offen/shared - steps: - - checkout: - path: ~/offen - - run: - name: Install dependencies - command: go get ./... - - run: - name: Run tests - command: make test - vault: docker: - image: circleci/node:10-browsers diff --git a/Makefile b/Makefile index aa9a1ab..5a317c7 100644 --- a/Makefile +++ b/Makefile @@ -25,4 +25,8 @@ bootstrap: @echo "Bootstrapping Accounts service ..." @docker-compose run accounts make bootstrap -.PHONY: setup bootstrap +build: + @docker build -t offen-server:latest -f build/server/Dockerfile . + @docker build -t offen-proxy:latest -f build/proxy/Dockerfile . + +.PHONY: setup bootstrap build diff --git a/build/docker-compose.yml b/build/docker-compose.yml index b45db47..0eef43d 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -2,15 +2,23 @@ version: '3' services: proxy: - build: - context: './..' - dockerfile: build/proxy/Dockerfile + image: offen-proxy:latest ports: - 3000:80 depends_on: - server server: - build: - context: './..' - dockerfile: build/server/Dockerfile + image: offen-server:latest + environment: + POSTGRES_CONNECTION_STRING: postgres://postgres:develop@server_database:5432/postgres?sslmode=disable + COOKIE_EXCHANGE_SECRET: Wsttdo4Z3mXV5sTc + EVENT_RETENTION_PERIOD: 4464h + ACCOUNT_USER_EMAIL_SALT: UwBkP24HCUYqy0Eq + depends_on: + - server_database + + server_database: + image: postgres:11.2 + environment: + POSTGRES_PASSWORD: develop diff --git a/build/proxy/Dockerfile b/build/proxy/Dockerfile index 40dd5b2..6d4d0da 100644 --- a/build/proxy/Dockerfile +++ b/build/proxy/Dockerfile @@ -50,8 +50,8 @@ RUN make publish FROM nginx:1.17-alpine COPY --from=homepage /code/homepage/output /www/data +COPY --from=script /code/script/dist /www/data COPY --from=auditorium /code/auditorium/dist /www/data/auditorium -COPY --from=script /code/script/dist /www/data/script COPY --from=vault /code/vault/dist /www/data/vault COPY ./build/proxy/nginx.conf /etc/nginx/nginx.conf diff --git a/build/proxy/nginx.conf b/build/proxy/nginx.conf index ae70f49..0e658c5 100644 --- a/build/proxy/nginx.conf +++ b/build/proxy/nginx.conf @@ -1,8 +1,12 @@ events {} http { - include /etc/nginx/mime.types; + gzip on; + gzip_comp_level 2; + gzip_min_length 1000; + gzip_proxied expired no-cache no-store private auth; + include mime.types; upstream server { server server:3000; } @@ -16,6 +20,12 @@ http { proxy_pass http://server; proxy_redirect off; rewrite ^/api(.*)$ $1 break; + proxy_hide_header Content-Type; + add_header Content-Type "application/json"; + } + + location /auditorium/ { + try_files $uri $uri/ /auditorium/index.html; } } } diff --git a/docker-compose.yml b/docker-compose.yml index ed4d640..fc95272 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,6 @@ services: - ./styles:/code/styles ports: - 8080:80 - - 8081:81 depends_on: - homepage - server @@ -26,7 +25,6 @@ services: - ./bootstrap.yml:/offen/server/bootstrap.yml - serverdeps:/go/pkg/mod environment: - CORS_ORIGIN: http://localhost:9977 POSTGRES_CONNECTION_STRING: postgres://postgres:develop@server_database:5432/postgres?sslmode=disable PORT: 8080 DEVELOPMENT: '1' @@ -61,8 +59,6 @@ services: - .:/offen - scriptdeps:/offen/script/node_modules command: npm start -- --port 9966 - environment: - - VAULT_HOST=/vault/ auditorium: build: @@ -73,8 +69,6 @@ services: - .:/offen - auditoriumdeps:/offen/auditorium/node_modules command: npm start -- --port 9955 - environment: - - VAULT_HOST=/vault/ homepage: build: diff --git a/homepage/content/pages/auditorium.md b/homepage/content/pages/auditorium.md deleted file mode 100644 index 463761a..0000000 --- a/homepage/content/pages/auditorium.md +++ /dev/null @@ -1,5 +0,0 @@ -Title: Auditorium | offen -description: offen is a free and open source analytics software for websites and web applications that allows respectful handling of data. -save_as: auditorium/index.html -href: /auditorium/ -template: auditorium diff --git a/homepage/publishconf.py b/homepage/publishconf.py index e442015..b28fdae 100644 --- a/homepage/publishconf.py +++ b/homepage/publishconf.py @@ -11,8 +11,8 @@ sys.path.append(os.curdir) from pelicanconf import * # If your site is available via HTTPS, make sure SITEURL begins with https:// -SITEURL = 'https://www.offen.dev' -RELATIVE_URLS = False +SITEURL = os.environ.get('PELICAN_SITEURL', 'https://www.offen.dev') +# RELATIVE_URLS = True FEED_ALL_ATOM = 'feeds/all.atom.xml' CATEGORY_FEED_ATOM = 'feeds/{slug}.atom.xml' diff --git a/homepage/theme/templates/auditorium.html b/homepage/theme/templates/auditorium.html deleted file mode 100644 index 997789b..0000000 --- a/homepage/theme/templates/auditorium.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-{% endblock %} - -{% block scripts %} - {{ super () }} - -{% endblock %} diff --git a/homepage/theme/templates/base.html b/homepage/theme/templates/base.html index 26b3792..160e6bd 100644 --- a/homepage/theme/templates/base.html +++ b/homepage/theme/templates/base.html @@ -20,7 +20,7 @@ {% assets filters="cssmin", output="css/style.min.css", "css/normalize.css", "css/fonts.css", "css/style.css" %} - + {% endassets %} {% if OFFEN_ACCOUNT_ID %} @@ -138,7 +138,7 @@ {% block scripts %} {% assets filters="rjsmin", output="scripts/packed.js", "scripts/jquery-3.4.1.min.js", "scripts/menu.js", "scripts/fade.js" %} - + {% endassets %} {% endblock %} diff --git a/shared/Makefile b/shared/Makefile deleted file mode 100644 index c56d5e2..0000000 --- a/shared/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -test: - @go test -race -cover ./... diff --git a/shared/README.md b/shared/README.md deleted file mode 100644 index c124d27..0000000 --- a/shared/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# shared - -`shared` contains Go packages shared across applications. Consumers need to symlink the folder into their app directory. - ---- - -## Development - -### Commands - -#### Run the tests - -``` -make test -``` diff --git a/shared/http/errors.go b/shared/http/errors.go deleted file mode 100644 index f095011..0000000 --- a/shared/http/errors.go +++ /dev/null @@ -1,24 +0,0 @@ -package http - -import ( - "encoding/json" - "net/http" -) - -type errorResponse struct { - Error string `json:"error"` - Status int `json:"status"` -} - -func jsonError(err error, status int) []byte { - r := errorResponse{err.Error(), status} - b, _ := json.Marshal(r) - return b -} - -// RespondWithJSONError writes the given error to the given response writer -// while wrapping it into a JSON payload. -func RespondWithJSONError(w http.ResponseWriter, err error, status int) { - w.WriteHeader(status) - w.Write(jsonError(err, status)) -} diff --git a/shared/http/errors_test.go b/shared/http/errors_test.go deleted file mode 100644 index 6923218..0000000 --- a/shared/http/errors_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package http - -import ( - "errors" - "net/http" - "net/http/httptest" - "testing" -) - -func TestRespondWithJSONError(t *testing.T) { - w := httptest.NewRecorder() - RespondWithJSONError(w, errors.New("does not work"), http.StatusInternalServerError) - if w.Code != http.StatusInternalServerError { - t.Errorf("Unexpected status code %d", w.Code) - } - if w.Body.String() != `{"error":"does not work","status":500}` { - t.Errorf("Unexpected response body %s", w.Body.String()) - } -} diff --git a/shared/http/jwt.go b/shared/http/jwt.go deleted file mode 100644 index bb2dfdd..0000000 --- a/shared/http/jwt.go +++ /dev/null @@ -1,174 +0,0 @@ -package http - -import ( - "context" - "crypto/x509" - "encoding/json" - "encoding/pem" - "errors" - "fmt" - "net/http" - "strings" - "time" - - "github.com/lestrrat-go/jwx/jwa" - "github.com/lestrrat-go/jwx/jwt" -) - -type contextKey string - -// ClaimsContextKey will be used to attach a JWT claim to a request context -const ClaimsContextKey contextKey = "claims" - -// JWTProtect uses the public key located at the given URL to check if the -// cookie value is signed properly. In case yes, the JWT claims will be added -// to the request context -func JWTProtect(keyURL, cookieName, headerName string, authorizer func(*http.Request, map[string]interface{}) error, cache Cache) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var jwtValue string - if authCookie, err := r.Cookie(cookieName); err == nil { - jwtValue = authCookie.Value - } else { - if header := r.Header.Get(headerName); header != "" { - jwtValue = header - } - } - if jwtValue == "" { - RespondWithJSONError(w, errors.New("jwt: could not infer JWT value from cookie or header"), http.StatusForbidden) - return - } - - var keys [][]byte - if cache != nil { - lookup, lookupErr := cache.Get() - if lookupErr == nil { - keys = lookup - } - } - if keys == nil { - var keysErr error - keys, keysErr = fetchKeys(keyURL) - if keysErr != nil { - RespondWithJSONError(w, fmt.Errorf("jwt: error fetching keys: %v", keysErr), http.StatusInternalServerError) - return - } - if cache != nil { - cache.Set(keys) - } - } - - var token *jwt.Token - var tokenErr error - // the response can contain multiple keys to try as some of them - // might have been retired with signed tokens still in use until - // their expiry - for _, key := range keys { - token, tokenErr = tryParse(key, jwtValue) - if tokenErr == nil { - break - } - } - - if tokenErr != nil { - RespondWithJSONError(w, fmt.Errorf("jwt: error verifying token signature: %v", tokenErr), http.StatusForbidden) - return - } - - if err := token.Verify(jwt.WithAcceptableSkew(0)); err != nil { - RespondWithJSONError(w, fmt.Errorf("jwt: error verifying token claims: %v", err), http.StatusForbidden) - return - } - - privKey, _ := token.Get("priv") - claims, _ := privKey.(map[string]interface{}) - if err := authorizer(r, claims); err != nil { - RespondWithJSONError(w, fmt.Errorf("jwt: token claims do not allow the requested operation: %v", err), http.StatusForbidden) - return - } - r = r.WithContext(context.WithValue(r.Context(), ClaimsContextKey, claims)) - next.ServeHTTP(w, r) - }) - } -} - -func tryParse(key []byte, tokenValue string) (*jwt.Token, error) { - keyBytes, _ := pem.Decode([]byte(key)) - if keyBytes == nil { - return nil, errors.New("jwt: no PEM block found in given key") - } - - pubKey, parseErr := x509.ParsePKCS1PublicKey(keyBytes.Bytes) - if parseErr != nil { - return nil, fmt.Errorf("jwt: error parsing key: %v", parseErr) - } - - token, jwtErr := jwt.ParseVerify(strings.NewReader(tokenValue), jwa.RS256, pubKey) - if jwtErr != nil { - return nil, fmt.Errorf("jwt: error parsing token: %v", jwtErr) - } - return token, nil -} - -type keyResponse struct { - Keys []string `json:"keys"` -} - -func fetchKeys(keyURL string) ([][]byte, error) { - fetchRes, fetchErr := http.Get(keyURL) - if fetchErr != nil { - return nil, fetchErr - } - defer fetchRes.Body.Close() - payload := keyResponse{} - if err := json.NewDecoder(fetchRes.Body).Decode(&payload); err != nil { - return nil, err - } - - asBytes := [][]byte{} - for _, key := range payload.Keys { - asBytes = append(asBytes, []byte(key)) - } - return asBytes, nil -} - -// Cache can be implemented by consumers in order to define how requests -// for public keys are being cached. For most use cases, the default cache -// supplied by this package will suffice. -type Cache interface { - Get() ([][]byte, error) - Set([][]byte) -} - -type defaultCache struct { - value *[][]byte - expires time.Duration - deadline time.Time -} - -// DefaultCacheExpiry should be used by cache instantiations without -// any particular requirements. -const DefaultCacheExpiry = time.Minute * 15 - -// ErrNoCache is returned on a cache lookup that did not yield a result -var ErrNoCache = errors.New("nothing found in cache") - -func (c *defaultCache) Get() ([][]byte, error) { - if c.value != nil && time.Now().Before(c.deadline) { - return *c.value, nil - } - return nil, ErrNoCache -} - -func (c *defaultCache) Set(value [][]byte) { - c.deadline = time.Now().Add(c.expires) - c.value = &value -} - -// NewDefaultKeyCache creates a simple cache that will hold a single -// value for the given expiration time -func NewDefaultKeyCache(expires time.Duration) Cache { - return &defaultCache{ - expires: expires, - } -} diff --git a/shared/http/jwt_test.go b/shared/http/jwt_test.go deleted file mode 100644 index 4826efd..0000000 --- a/shared/http/jwt_test.go +++ /dev/null @@ -1,279 +0,0 @@ -package http - -import ( - "crypto/x509" - "encoding/pem" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/lestrrat-go/jwx/jwa" - - "github.com/lestrrat-go/jwx/jwt" -) - -const publicKey = ` ------BEGIN RSA PUBLIC KEY----- -MIIBCgKCAQEAl2ifzOsh6AMRHZBe8xycvk01s3EAGT12WbgV9Z7b420Dj3NXrkns -N/jvBbtO9cjQg4WM7NPZLs+ZutRkHCtMxt7vB0kjOYetLPcGdObsVBB5k1jvwvsJ -HkcfmSsZdrV0Lz2Yxuf6ADWkxBqAY3GsS0zW0A2nIMc+41ZxqsZa3ProsKJxecRX -SSZMpZtqCGt/S83Rek4eAahllcWfZpQmoEk7usLuUl5tH2TmaW3e5lo0JNfdwcq5 -PMCa8WSZBFH3YzVttB8rbe7a7336wL2NJQFz6dswL5X1dECYpZ5TRtNgzQYa4V0W -AeICq+EzigaTxrjDHc5urHqEosz1le7O4QIDAQAB ------END RSA PUBLIC KEY----- -` - -const privateKey = ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCXaJ/M6yHoAxEd -kF7zHJy+TTWzcQAZPXZZuBX1ntvjbQOPc1euSew3+O8Fu071yNCDhYzs09kuz5m6 -1GQcK0zG3u8HSSM5h60s9wZ05uxUEHmTWO/C+wkeRx+ZKxl2tXQvPZjG5/oANaTE -GoBjcaxLTNbQDacgxz7jVnGqxlrc+uiwonF5xFdJJkylm2oIa39LzdF6Th4BqGWV -xZ9mlCagSTu6wu5SXm0fZOZpbd7mWjQk193Byrk8wJrxZJkEUfdjNW20Hytt7trv -ffrAvY0lAXPp2zAvlfV0QJilnlNG02DNBhrhXRYB4gKr4TOKBpPGuMMdzm6seoSi -zPWV7s7hAgMBAAECggEAYoPOxjSP4ThtoIDZZvHNAv2V3WW/HK0jHolqsGBmznmW -AXaZLGwo6NpuG5qea8n38jupUEcfXxfw/OFJKhL6Z8OSX3k1FC+1fDZW2yWNy7zU -fg02I/XXHv5EDxM+BEFYkYxQpcs2nYBJ7tcXhpzl8DDU7JaVkfxSbPVIDEf3wyP2 -k6DjYEeAj7uAsp50/32H9zhlJP/cFZaPiyFYy9/gOmDenrPVyJ/f7iQYNwYAwdbt -yfp11Wd5BpePR58+YXICE8oBtzHvB50akK6RZULC3xHVxLQQ1bSxx6vnttxw5RW+ -QRHTVWtRyWiKe/l5jMvVSUo5XCLqsL2iXfR4bz6hyQKBgQDJQVWEGHyD6JyaN/6i -5M5+O/YTbzMBgt1JAuVR2c0HYE4LpgrX4cA4kT3Bsa/Z9o0uuWVuxVab5gLsjsDu -EI4o+HJQ3pl4LF9xqrndTdybwmZAT6jv3rM/VGfaCDzXCPzVx169I+WsfkyGW7Tr -Cj4KDZyykruG/9OrpN1Aeq9a3wKBgQDAmCnQHLEPvZdezJGBc34HjZntrbW67iFB -L0waCGWydyunYmzfja1FSvlSoziZdqoq0N4+uBQFPZIlERvq0zMgIzNFvxt/WlFV -kQcBV24MNa8dtd+P7GDY8TzTfYBeXwJoi5c59sWLzSwpTlw0rI+ZvuA66eEF7xih -eWw3k1jOPwKBgQCKf8DHGEbQTEtBQlmlZkrIyqDs/PCgEJwSe8Cu1HFpqxfqokkC -CiTLiQB0BMEdAbRlPEcWtQ2GWgMXIqKY8qGyhk+9YYNCFV9VjQU9zDCOrHjLt0Zu -VNcMNR0HCfY8kb3VrM+A4GxVidFGAWR+/9xz9KwqpBoTrIjRrbJphkSZBwKBgBMs -0zTqNmLH0JNasL3/vrOH0KSOYAKdhOgVinEpFt7+6HTA4vAbDf5RKaOlppP48ZZT -t1ztPOkMqUlRe8MUhgmUF53BGj7CwkhPqS/kAYvrqGS/3+NXeIkA87pmy2oZ8YZx -J3xY6nAx3Ey8hYelCqMXEwIqmQHbPUuOaEzcOcJHAoGANw0TRha2YSbUqS3HiGR/ -Jm0lNfeLc3cYlEq0ggRDPLD11485rxVKnaKVHGPYW28OQ4jA+GCc7VZCfPV8VQXW -6b+jUnnBwu/KuYvGMee/xJv1c4MTG54mR9UrUt+R80S0OplpcYkcQnft2Bi+AZ1h -5aZTE7XIouXCYiMKPl4AMtI= ------END PRIVATE KEY----- -` - -func TestJWTProtect(t *testing.T) { - tests := []struct { - name string - cookie *http.Cookie - headers *http.Header - server *httptest.Server - authorizer func(r *http.Request, claims map[string]interface{}) error - expectedStatusCode int - }{ - { - "no cookie", - nil, - nil, - nil, - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusForbidden, - }, - { - "bad server", - &http.Cookie{ - Name: "auth", - Value: "irrelevantgibberish", - }, - nil, - nil, - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusInternalServerError, - }, - { - "non-json response", - &http.Cookie{ - Name: "auth", - Value: "irrelevantgibberish", - }, - nil, - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("here's some bytes 4 y'all")) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusInternalServerError, - }, - { - "bad key value", - &http.Cookie{ - Name: "auth", - Value: "irrelevantgibberish", - }, - nil, - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"keys":["not really a key","me neither"]}`)) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusForbidden, - }, - { - "invalid key", - &http.Cookie{ - Name: "auth", - Value: "irrelevantgibberish", - }, - nil, - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"keys":["-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCATZAMIIBCgKCAQEA2yUfHH6SRYKvBTemrefi\nHk4L4qkcc4skl4QCaHOkfgA4VcGKG2nXysYuZK7AzNOcHQVi+e4BwN+BfIZtwEU5\n7Ogctb5eg8ksxxLjS7eSRfQIvPGfAbJ12R9OoOWcue/CdUy/YMec4R/o4+tZ45S6\nQQWIMhLqYljw+s1Runda3K8Q8lOdJ4yEZckXaZr1waNJikC7oGpT7ClAgdbvWIbo\nN18G1OluRn+3WNdcN6V+vIj8c9dGs92bgTPX4cn3RmB/80BDfzeFiPMRw5xaq66F\n42zXzllkTqukQPk2wmO5m9pFy0ciRve+awfgbTtZRZOEpTSWLbbpOfd4RQ5YqDWJ\nmQIDAQAB\n-----END PUBLIC KEY-----"]}`)) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusForbidden, - }, - { - "valid key, bad auth token", - &http.Cookie{ - Name: "auth", - Value: "irrelevantgibberish", - }, - nil, - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte( - fmt.Sprintf(`{"keys":["%s"]}`, strings.ReplaceAll(publicKey, "\n", `\n`)), - )) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusForbidden, - }, - { - "valid key, valid token", - &http.Cookie{ - Name: "auth", - Value: (func() string { - token := jwt.New() - token.Set("exp", time.Now().Add(time.Hour)) - keyBytes, _ := pem.Decode([]byte(privateKey)) - privKey, _ := x509.ParsePKCS8PrivateKey(keyBytes.Bytes) - b, _ := token.Sign(jwa.RS256, privKey) - return string(b) - })(), - }, - nil, - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte( - fmt.Sprintf(`{"keys":["%s"]}`, strings.ReplaceAll(publicKey, "\n", `\n`)), - )) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusOK, - }, - { - "ok token in headers", - nil, - (func() *http.Header { - token := jwt.New() - token.Set("exp", time.Now().Add(time.Hour)) - keyBytes, _ := pem.Decode([]byte(privateKey)) - privKey, _ := x509.ParsePKCS8PrivateKey(keyBytes.Bytes) - b, _ := token.Sign(jwa.RS256, privKey) - return &http.Header{ - "X-RPC-Authentication": []string{string(b)}, - } - })(), - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte( - fmt.Sprintf(`{"keys":["%s"]}`, strings.ReplaceAll(publicKey, "\n", `\n`)), - )) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusOK, - }, - { - "bad token in headers", - nil, - (func() *http.Header { - return &http.Header{ - "X-RPC-Authentication": []string{"nilly willy"}, - } - })(), - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte( - fmt.Sprintf(`{"keys":["%s"]}`, strings.ReplaceAll(publicKey, "\n", `\n`)), - )) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusForbidden, - }, - { - "authorizer rejects", - &http.Cookie{ - Name: "auth", - Value: (func() string { - token := jwt.New() - token.Set("exp", time.Now().Add(time.Hour)) - token.Set("priv", map[string]interface{}{"ok": false}) - keyBytes, _ := pem.Decode([]byte(privateKey)) - privKey, _ := x509.ParsePKCS8PrivateKey(keyBytes.Bytes) - b, _ := token.Sign(jwa.RS256, privKey) - return string(b) - })(), - }, - nil, - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte( - fmt.Sprintf(`{"keys":["%s"]}`, strings.ReplaceAll(publicKey, "\n", `\n`)), - )) - })), - func(r *http.Request, claims map[string]interface{}) error { - if claims["ok"] == true { - return nil - } - return errors.New("expected ok to be true") - }, - http.StatusForbidden, - }, - { - "valid key, expired token", - &http.Cookie{ - Name: "auth", - Value: (func() string { - token := jwt.New() - token.Set("exp", time.Now().Add(-time.Hour)) - keyBytes, _ := pem.Decode([]byte(privateKey)) - privKey, _ := x509.ParsePKCS8PrivateKey(keyBytes.Bytes) - b, _ := token.Sign(jwa.RS256, privKey) - return string(b) - })(), - }, - nil, - httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte( - fmt.Sprintf(`{"keys":["%s"]}`, strings.ReplaceAll(publicKey, "\n", `\n`)), - )) - })), - func(r *http.Request, claims map[string]interface{}) error { return nil }, - http.StatusForbidden, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - var url string - if test.server != nil { - url = test.server.URL - } - wrappedHandler := JWTProtect(url, "auth", "X-RPC-Authentication", test.authorizer, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) - })) - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - if test.cookie != nil { - r.AddCookie(test.cookie) - } - if test.headers != nil { - for key, value := range *test.headers { - r.Header.Add(key, value[0]) - } - } - wrappedHandler.ServeHTTP(w, r) - if w.Code != test.expectedStatusCode { - t.Errorf("Unexpected status code %v", w.Code) - } - }) - } -} diff --git a/shared/http/middleware.go b/shared/http/middleware.go deleted file mode 100644 index d1dc3ee..0000000 --- a/shared/http/middleware.go +++ /dev/null @@ -1,64 +0,0 @@ -package http - -import ( - "context" - "errors" - "net/http" -) - -// CorsMiddleware ensures the wrapped handler will respond with proper CORS -// headers using the given origin. -func CorsMiddleware(origin string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Credentials", "true") - w.Header().Set("Access-Control-Allow-Methods", "POST,GET") - w.Header().Set("Access-Control-Allow-Origin", origin) - next.ServeHTTP(w, r) - }) - } -} - -// ContentTypeMiddleware ensuresthe wrapped handler will respond with a -// content type header of the given value. -func ContentTypeMiddleware(contentType string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("Content-Type", contentType) - next.ServeHTTP(w, r) - }) - } -} - -// OptoutMiddleware drops all requests to the given handler that are sent with -// a cookie of the given name, -func OptoutMiddleware(cookieName string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if _, err := r.Cookie(cookieName); err == nil { - w.WriteHeader(http.StatusNoContent) - return - } - next.ServeHTTP(w, r) - }) - } -} - -// UserCookieMiddleware ensures a cookie of the given name is present and -// attaches its value to the request's context using the given key, before -// passing it on to the wrapped handler. -func UserCookieMiddleware(cookieKey string, contextKey interface{}) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - c, err := r.Cookie(cookieKey) - if err != nil { - RespondWithJSONError(w, errors.New("user cookie: received no or blank identifier"), http.StatusBadRequest) - return - } - r = r.WithContext( - context.WithValue(r.Context(), contextKey, c.Value), - ) - next.ServeHTTP(w, r) - }) - } -} diff --git a/shared/http/middleware_test.go b/shared/http/middleware_test.go deleted file mode 100644 index 633d44b..0000000 --- a/shared/http/middleware_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package http - -import ( - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestCorsMiddleware(t *testing.T) { - t.Run("default", func(t *testing.T) { - wrapped := CorsMiddleware("https://www.example.net")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) - })) - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - wrapped.ServeHTTP(w, r) - if h := w.Header().Get("Access-Control-Allow-Origin"); h != "https://www.example.net" { - t.Errorf("Unexpected header value %v", h) - } - }) -} - -func TestContentTypeMiddleware(t *testing.T) { - t.Run("default", func(t *testing.T) { - wrapped := ContentTypeMiddleware("application/json")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) - })) - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - wrapped.ServeHTTP(w, r) - if h := w.Header().Get("Content-Type"); h != "application/json" { - t.Errorf("Unexpected header value %v", h) - } - }) -} - -func TestOptoutMiddleware(t *testing.T) { - wrapped := OptoutMiddleware("optout")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("hey there")) - })) - t.Run("with header", func(t *testing.T) { - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - r.AddCookie(&http.Cookie{ - Name: "optout", - }) - wrapped.ServeHTTP(w, r) - - if w.Code != http.StatusNoContent { - t.Errorf("Unexpected status code %d", w.Code) - } - - if w.Body.String() != "" { - t.Errorf("Unexpected response body %s", w.Body.String()) - } - }) - t.Run("without header", func(t *testing.T) { - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - wrapped.ServeHTTP(w, r) - - if w.Code != http.StatusOK { - t.Errorf("Unexpected status code %d", w.Code) - } - - if w.Body.String() != "hey there" { - t.Errorf("Unexpected response body %s", w.Body.String()) - } - }) -} - -func TestUserCookieMiddleware(t *testing.T) { - wrapped := UserCookieMiddleware("user", 1)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - value := r.Context().Value(1) - fmt.Fprintf(w, "value is %v", value) - })) - t.Run("no cookie", func(t *testing.T) { - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - wrapped.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { - t.Errorf("Unexpected status code %v", w.Code) - } - if !strings.Contains(w.Body.String(), "received no or blank identifier") { - t.Errorf("Unexpected body %s", w.Body.String()) - } - }) - - t.Run("no value", func(t *testing.T) { - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - wrapped.ServeHTTP(w, r) - r.AddCookie(&http.Cookie{ - Name: "user", - Value: "", - }) - if w.Code != http.StatusBadRequest { - t.Errorf("Unexpected status code %v", w.Code) - } - if !strings.Contains(w.Body.String(), "received no or blank identifier") { - t.Errorf("Unexpected body %s", w.Body.String()) - } - }) - - t.Run("ok", func(t *testing.T) { - w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodGet, "/", nil) - r.AddCookie(&http.Cookie{ - Name: "user", - Value: "token", - }) - wrapped.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Errorf("Unexpected status code %v", w.Code) - } - if w.Body.String() != "value is token" { - t.Errorf("Unexpected body %s", w.Body.String()) - } - }) -} diff --git a/styles/index.css b/styles/index.css deleted file mode 100644 index e69de29..0000000 From ca7323a456b45798dbadb5d6f3f9a508498ebe9e Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 5 Sep 2019 11:56:24 +0200 Subject: [PATCH 11/20] implement access control for account data --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc4960e..107b0a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -225,7 +225,5 @@ workflows: <<: *build_preconditions - packages: <<: *build_preconditions - - shared: - <<: *build_preconditions - deploy_homepage: <<: *deploy_preconditions From 16a731e3b23b1b7169037eafeaa188e89dab9822 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 5 Sep 2019 14:02:01 +0200 Subject: [PATCH 12/20] use base href for handling spa links --- Makefile | 5 ++++- README.md | 14 +------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 5a317c7..7510060 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,12 @@ + help: @echo " setup" - @echo " Build the containers and install dependencies." + @echo " Build the development containers and install dependencies." @echo " bootstrap" @echo " Set up keys and seed databases." @echo " IMPORTANT: this wipes any existing data in your local database." + @echo " build" + @echo " Build the production containers." setup: @docker-compose build diff --git a/README.md b/README.md index ccaa460..c3f77e2 100644 --- a/README.md +++ b/README.md @@ -37,19 +37,7 @@ You can test your setup by starting the application: $ docker-compose up ``` -which should enable you to access and use the `auditorium` - -### Developing the homepage - -In order to ease sharing of styles, the site is also part of this repository. It runs in a separate development environment: - -``` -$ cd homepage -$ make setup -$ docker-compose up -``` - -A live reloading development server will run on . +which should enable you to access and use the `auditorium` ### License From aa33a2110eff7185292e0adc10f5ffa8417241a4 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 5 Sep 2019 16:24:01 +0200 Subject: [PATCH 13/20] start collecting tools in single command --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fc95272..400001a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,9 +28,9 @@ services: POSTGRES_CONNECTION_STRING: postgres://postgres:develop@server_database:5432/postgres?sslmode=disable PORT: 8080 DEVELOPMENT: '1' - COOKIE_EXCHANGE_SECRET: Wsttdo4Z3mXV5sTc + COOKIE_EXCHANGE_SECRET: 3P+w6QetKO3Kn8h1YyRlCw== EVENT_RETENTION_PERIOD: 4464h - ACCOUNT_USER_EMAIL_SALT: UwBkP24HCUYqy0Eq + ACCOUNT_USER_EMAIL_SALT: S6ielTPzd/AGR88FI25c4Q== command: refresh run links: - server_database From 7c4b39e6d6baaa0916bb7d112d91056091e28378 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 5 Sep 2019 21:05:14 +0200 Subject: [PATCH 14/20] merge all ops tasks into single cli --- .npmrc | 1 - Makefile | 11 ++++------- docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 .npmrc diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 43c97e7..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -package-lock=false diff --git a/Makefile b/Makefile index 7510060..a25adf7 100644 --- a/Makefile +++ b/Makefile @@ -10,26 +10,23 @@ help: setup: @docker-compose build - @docker-compose run accounts pip install --user -r requirements.txt -r requirements-dev.txt @docker-compose run script npm install @docker-compose run vault npm install @docker-compose run auditorium npm install @docker-compose run server go mod download - @docker-compose run kms go mod download @echo "Successfully built containers and installed dependencies." @echo "If this is your initial setup, you can run 'make bootstrap' next" @echo "to create the needed local keys and seed the database." bootstrap: - @echo "Bootstrapping KMS service ..." - @docker-compose run kms make bootstrap @echo "Bootstrapping Server service ..." @docker-compose run server make bootstrap - @echo "Bootstrapping Accounts service ..." - @docker-compose run accounts make bootstrap build: @docker build -t offen-server:latest -f build/server/Dockerfile . @docker build -t offen-proxy:latest -f build/proxy/Dockerfile . -.PHONY: setup bootstrap build +secret: + @docker-compose run server make secret + +.PHONY: setup bootstrap build secret diff --git a/docker-compose.yml b/docker-compose.yml index 400001a..aba0902 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: DEVELOPMENT: '1' COOKIE_EXCHANGE_SECRET: 3P+w6QetKO3Kn8h1YyRlCw== EVENT_RETENTION_PERIOD: 4464h - ACCOUNT_USER_EMAIL_SALT: S6ielTPzd/AGR88FI25c4Q== + ACCOUNT_USER_EMAIL_SALT: JuhbRA4lCdo8rt5qVdLpk3== command: refresh run links: - server_database From b4e6faff48b126a71a5012a62489c7c42212078a Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 9 Sep 2019 20:01:22 +0200 Subject: [PATCH 15/20] build docker images in ci --- .circleci/config.yml | 124 +++++++++++++---------------- .ebignore | 2 + .elasticbeanstalk/config.yml | 17 ++++ Dockerrun.aws.json | 31 ++++++++ Makefile | 4 +- build/docker-compose.yml | 24 ------ homepage/theme/templates/base.html | 2 +- 7 files changed, 109 insertions(+), 95 deletions(-) create mode 100644 .ebignore create mode 100644 .elasticbeanstalk/config.yml create mode 100644 Dockerrun.aws.json delete mode 100644 build/docker-compose.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 107b0a0..909bc37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,16 +1,6 @@ version: 2 -production_env: &production_env - environment: - - SERVER_HOST=https://server-alpha.offen.dev - - SCRIPT_HOST=https://script-alpha.offen.dev - - AUDITORIUM_HOST=https://auditorium-alpha.offen.dev - - VAULT_HOST=https://vault-alpha.offen.dev - - HOMEPAGE_HOST=https://www.offen.dev - - NODE_ENV=production - - SECRET_ID_SERVER_CONNECTION_STRING=alpha/server/postgresConnectionString - -deploy_preconditions: &deploy_preconditions +build_preconditions: &build_preconditions requires: - server - vault @@ -21,10 +11,12 @@ deploy_preconditions: &deploy_preconditions branches: only: /^master$/ -build_preconditions: &build_preconditions +deploy_preconditions: &deploy_preconditions + requires: + - build filters: branches: - ignore: gh-pages + only: /^master$/ jobs: server: @@ -156,74 +148,70 @@ jobs: name: Run tests command: npm test - deploy_homepage: + build: docker: - - image: circleci/python:3.6-node - working_directory: ~/offen/homepage - environment: - - SOURCE_BRANCH: master - - TARGET_BRANCH: gh-pages + - image: docker:18-git + working_directory: ~/offen steps: - - checkout: - path: ~/offen + - checkout + - setup_remote_docker - restore_cache: - key: offen-homepage-{{ checksum "requirements.txt" }} - - run: - name: Install dependencies - command: | - python -m venv venv - source venv/bin/activate - pip install -r requirements.txt - - save_cache: + keys: + - v1-{{ .Branch }} paths: - - ~/offen/homepage/venv - key: offen-homepage-{{ checksum "requirements.txt" }} + - /caches/proxy.tar + - /caches/server.tar - run: - name: Install image optimization deps + name: Load Docker image layer cache command: | - sudo npm install svgo -g - sudo apt-get install libjpeg-progs optipng + set +o pipefail + docker load -i /caches/app.tar | true - run: - name: Deploy + name: Build application Docker image command: | - source venv/bin/activate + docker build --cache-from=offen/proxy -t offen/proxy:latest -f build/proxy/Dockerfile . + docker build --cache-from=offen/server -t offen/server:latest -f build/server/Dockerfile . + - run: + name: Save Docker image layer cache + command: | + mkdir -p /caches + docker save -o /caches/proxy.tar offen/proxy + docker save -o /caches/server.tar offen/server + - save_cache: + key: v1-{{ .Branch }}-{{ epoch }} + paths: + - /caches/app.tar + - deploy: + name: Push application Docker image + command: | + echo "$DOCKER_PASS" | docker login --username $DOCKER_USER --password-stdin + docker push offen/server:latest + docker push offen/proxy:latest - git config --global user.email $GH_EMAIL - git config --global user.name $GH_NAME - - git clone $CIRCLE_REPOSITORY_URL out - - cd out - git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH - git rm -rf . - cd .. - - make publish - - cp -a output/. out/. - - mkdir -p out/.circleci && cp -a ./../.circleci/. out/.circleci/. - cp CNAME out/CNAME - cd out - - git add -A - git commit -m "Automated deployment to GitHub Pages: ${CIRCLE_SHA1}" --allow-empty - - git push origin $TARGET_BRANCH + deploy: + working_directory: ~/offen + docker: + - image: circleci/python:3.6 + steps: + - checkout + - run: + name: Installing deployment dependencies + command: | + sudo pip install awsebcli --upgrade + - run: + name: Deploying + command: eb deploy staging workflows: version: 2 test_build_deploy: jobs: - - server: + - server + - vault + - script + - auditorium + - packages + - build: <<: *build_preconditions - - vault: - <<: *build_preconditions - - script: - <<: *build_preconditions - - auditorium: - <<: *build_preconditions - - packages: - <<: *build_preconditions - - deploy_homepage: + - deploy: <<: *deploy_preconditions diff --git a/.ebignore b/.ebignore new file mode 100644 index 0000000..5712733 --- /dev/null +++ b/.ebignore @@ -0,0 +1,2 @@ +*.* +!Dockerrun.aws.json diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml new file mode 100644 index 0000000..d90868c --- /dev/null +++ b/.elasticbeanstalk/config.yml @@ -0,0 +1,17 @@ +branch-defaults: + default: + environment: staging + group_suffix: null +global: + application_name: offen + branch: null + default_ec2_keyname: aws-eb + default_platform: Multi-container Docker 18.06.1-ce (Generic) + default_region: eu-central-1 + include_git_submodules: true + instance_profile: null + platform_name: null + platform_version: null + repository: null + sc: null + workspace_type: Application diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json new file mode 100644 index 0000000..fce0391 --- /dev/null +++ b/Dockerrun.aws.json @@ -0,0 +1,31 @@ +{ + "AWSEBDockerrunVersion": 2, + "volumes": [], + "containerDefinitions": [ + { + "name": "proxy", + "image": "offen/proxy:latest", + "essential": true, + "memory": 128, + "portMappings": [ + { + "hostPort": 80, + "containerPort": 80 + } + ], + "dependsOn": [ + { + "containerName": "server", + "condition": "START" + } + ], + "links": ["server"] + }, + { + "name": "server", + "image": "offen/server:latest", + "essential": true, + "memory": 128 + } + ] +} diff --git a/Makefile b/Makefile index a25adf7..2ae75e7 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,8 @@ bootstrap: @docker-compose run server make bootstrap build: - @docker build -t offen-server:latest -f build/server/Dockerfile . - @docker build -t offen-proxy:latest -f build/proxy/Dockerfile . + @docker build -t offen/server:latest -f build/server/Dockerfile . + @docker build -t offen/proxy:latest -f build/proxy/Dockerfile . secret: @docker-compose run server make secret diff --git a/build/docker-compose.yml b/build/docker-compose.yml deleted file mode 100644 index 0eef43d..0000000 --- a/build/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '3' - -services: - proxy: - image: offen-proxy:latest - ports: - - 3000:80 - depends_on: - - server - - server: - image: offen-server:latest - environment: - POSTGRES_CONNECTION_STRING: postgres://postgres:develop@server_database:5432/postgres?sslmode=disable - COOKIE_EXCHANGE_SECRET: Wsttdo4Z3mXV5sTc - EVENT_RETENTION_PERIOD: 4464h - ACCOUNT_USER_EMAIL_SALT: UwBkP24HCUYqy0Eq - depends_on: - - server_database - - server_database: - image: postgres:11.2 - environment: - POSTGRES_PASSWORD: develop diff --git a/homepage/theme/templates/base.html b/homepage/theme/templates/base.html index 160e6bd..4c29415 100644 --- a/homepage/theme/templates/base.html +++ b/homepage/theme/templates/base.html @@ -23,7 +23,7 @@ {% endassets %} {% if OFFEN_ACCOUNT_ID %} - + {% endif %} From cc97e2be9d5bc71928c6a13798e81178b43736fc Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 9 Sep 2019 21:43:41 +0200 Subject: [PATCH 16/20] add styles for auditorium --- .circleci/config.yml | 2 +- .elasticbeanstalk/config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 909bc37..5d617bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -200,7 +200,7 @@ jobs: sudo pip install awsebcli --upgrade - run: name: Deploying - command: eb deploy staging + command: eb deploy workflows: version: 2 diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml index d90868c..438573c 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.yml @@ -1,5 +1,5 @@ branch-defaults: - default: + master: environment: staging group_suffix: null global: From becb0bcef933eaebeacdf34f22df9252adb011a5 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 9 Sep 2019 21:45:05 +0200 Subject: [PATCH 17/20] update internal auditorium link --- homepage/content/pages/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homepage/content/pages/index.md b/homepage/content/pages/index.md index dc7dc0b..5cfaa7c 100644 --- a/homepage/content/pages/index.md +++ b/homepage/content/pages/index.md @@ -12,7 +12,7 @@ Usage metrics come with explanations about their meaning, relevance, usage and p __offen__ treats both users and operators as parties of equal importance. Users can expect full transparency and are encouraged to make *autonomous and informed decisions regarding the use of their data*. Operators are enabled to gain insights while respecting their users' privacy and data. -__offen__ is currently in the early stages of development and is applying for funds to sustain its development. An early alpha version is running on this site: you can *visit the [auditorium](https://auditorium-alpha.offen.dev){: target="_blank"}* to access your data. +__offen__ is currently in the early stages of development and is applying for funds to sustain its development. An early alpha version is running on this site: you can *visit the [auditorium](/auditorium/)* to access your data.
Deep dive From 5656a3df7d72e6e10435074317b7d1bec5428c08 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 9 Sep 2019 22:30:11 +0200 Subject: [PATCH 18/20] leverage layer caching for node deps --- .ebignore | 2 +- build/proxy/Dockerfile | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.ebignore b/.ebignore index 5712733..8f16f47 100644 --- a/.ebignore +++ b/.ebignore @@ -1,2 +1,2 @@ -*.* +* !Dockerrun.aws.json diff --git a/build/proxy/Dockerfile b/build/proxy/Dockerfile index 6d4d0da..a1f820d 100644 --- a/build/proxy/Dockerfile +++ b/build/proxy/Dockerfile @@ -1,34 +1,43 @@ FROM node:10 as auditorium -COPY ./auditorium /code/auditorium +COPY ./auditorium/package.json /code/deps/package.json COPY ./packages /code/packages +WORKDIR /code/deps +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +RUN npm install +COPY ./auditorium /code/auditorium COPY ./styles /code/styles COPY ./banner.txt /code/banner.txt WORKDIR /code/auditorium -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true -RUN npm install +RUN cp -a /code/deps/node_modules /code/auditorium/ ENV NODE_ENV production RUN npm run build FROM node:10 as script -COPY ./script /code/script +COPY ./script/package.json /code/deps/package.json COPY ./packages /code/packages -COPY ./banner.txt /code/banner.txt -WORKDIR /code/script +WORKDIR /code/deps ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true RUN npm install +COPY ./script /code/script +COPY ./banner.txt /code/banner.txt +WORKDIR /code/script +RUN cp -a /code/deps/node_modules /code/script/ ENV NODE_ENV production RUN npm run build FROM node:10 as vault -COPY ./vault /code/vault +COPY ./vault/package.json /code/deps/package.json COPY ./packages /code/packages -COPY ./banner.txt /code/banner.txt -WORKDIR /code/vault +WORKDIR /code/deps ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true RUN npm install +COPY ./vault /code/vault +COPY ./banner.txt /code/banner.txt +WORKDIR /code/vault +RUN cp -a /code/deps/node_modules /code/vault/ ENV NODE_ENV production RUN npm run build From 564ff3ad30c88544e76d218678bbed43c34bfa8c Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Tue, 10 Sep 2019 09:17:04 +0200 Subject: [PATCH 19/20] fix deletion of outdated user keys --- homepage/CNAME | 1 - homepage/pelicanconf.py | 1 - homepage/theme/static/scripts/fade.js | 8 ++++---- homepage/theme/static/scripts/jump.js | 0 4 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 homepage/CNAME delete mode 100644 homepage/theme/static/scripts/jump.js diff --git a/homepage/CNAME b/homepage/CNAME deleted file mode 100644 index 8f7cb2a..0000000 --- a/homepage/CNAME +++ /dev/null @@ -1 +0,0 @@ -www.offen.dev diff --git a/homepage/pelicanconf.py b/homepage/pelicanconf.py index 1fd3ff4..9a786bf 100644 --- a/homepage/pelicanconf.py +++ b/homepage/pelicanconf.py @@ -43,4 +43,3 @@ DIRECT_TEMPLATES = [] GITHUB_ORG = 'https://github.com/offen' CONTACT_EMAIL = 'mail@offen.dev' PATREON_URL = 'https://www.patreon.com/bePatron?u=21484999' -AUDITORIUM_SCRIPT = 'http://localhost:8080/auditorium/index.js' diff --git a/homepage/theme/static/scripts/fade.js b/homepage/theme/static/scripts/fade.js index 0490c5b..44b4f7e 100644 --- a/homepage/theme/static/scripts/fade.js +++ b/homepage/theme/static/scripts/fade.js @@ -1,10 +1,10 @@ ;(function ($) { $(document).ready(function () { $(window).scroll(function () { - var scrollProgress = parseInt($(window).scrollTop(), 10) - if ($(window).width() > 960) { - $('body.index .brand').css('opacity', Math.min(scrollProgress / 100, 1)) - } + if ($(window).width() > 960) { + var scrollProgress = parseInt($(window).scrollTop(), 10) + $('body.index .brand').css('opacity', Math.min(scrollProgress / 100, 1)) + } }) }) })(window.jQuery) diff --git a/homepage/theme/static/scripts/jump.js b/homepage/theme/static/scripts/jump.js deleted file mode 100644 index e69de29..0000000 From e40af3912a0e52a4e746aa99fc6f5b163e3bda8a Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 11 Sep 2019 12:44:51 +0200 Subject: [PATCH 20/20] configure production environment --- .circleci/config.yml | 14 ++++++++++---- .elasticbeanstalk/config.yml | 4 +++- .gitignore | 1 + ...errun.aws.json => Dockerrun.aws.json.production | 6 +++--- Makefile | 6 ++++-- README.md | 2 +- build/proxy/Dockerfile | 3 +++ homepage/publishconf.py | 2 +- 8 files changed, 26 insertions(+), 12 deletions(-) rename Dockerrun.aws.json => Dockerrun.aws.json.production (83%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d617bc..486a573 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,9 +23,10 @@ jobs: docker: - image: circleci/golang:1.12 environment: - - POSTGRES_CONNECTION_STRING=postgres://circle:test@localhost:5432/circle_test?sslmode=disable - - PORT=8080 - - EVENT_RETENTION_PERIOD=4464h + POSTGRES_CONNECTION_STRING: postgres://circle:test@localhost:5432/circle_test?sslmode=disable + PORT: 8080 + EVENT_RETENTION_PERIOD: 4464h + COOKIE_EXCHANGE_SECRET: VswgMshC4mPDfey8o+yScg== - image: circleci/postgres:11.2-alpine environment: - POSTGRES_USER=circle @@ -151,6 +152,9 @@ jobs: build: docker: - image: docker:18-git + environment: + SITEURL: https://www.offen.dev + DOCKER_TAGE: stable working_directory: ~/offen steps: - checkout @@ -200,7 +204,9 @@ jobs: sudo pip install awsebcli --upgrade - run: name: Deploying - command: eb deploy + command: | + cp Dockerrun.aws.json.production Dockerrun.aws.json + eb deploy workflows: version: 2 diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml index 438573c..38d4188 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.yml @@ -1,6 +1,8 @@ branch-defaults: + local-proxy: + environment: production master: - environment: staging + environment: production group_suffix: null global: application_name: offen diff --git a/.gitignore b/.gitignore index 440863e..f66fb9c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ package-lock.json venv/ bootstrap-alpha.yml +Dockerrun.aws.json diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json.production similarity index 83% rename from Dockerrun.aws.json rename to Dockerrun.aws.json.production index fce0391..0d984f7 100644 --- a/Dockerrun.aws.json +++ b/Dockerrun.aws.json.production @@ -4,7 +4,7 @@ "containerDefinitions": [ { "name": "proxy", - "image": "offen/proxy:latest", + "image": "offen/proxy:stable", "essential": true, "memory": 128, "portMappings": [ @@ -23,9 +23,9 @@ }, { "name": "server", - "image": "offen/server:latest", + "image": "offen/server:stable", "essential": true, - "memory": 128 + "memory": 256 } ] } diff --git a/Makefile b/Makefile index 2ae75e7..7e1cdca 100644 --- a/Makefile +++ b/Makefile @@ -22,9 +22,11 @@ bootstrap: @echo "Bootstrapping Server service ..." @docker-compose run server make bootstrap +DOCKER_IMAGE_TAG ?= latest + build: - @docker build -t offen/server:latest -f build/server/Dockerfile . - @docker build -t offen/proxy:latest -f build/proxy/Dockerfile . + @docker build -t offen/server:${DOCKER_IMAGE_TAG} -f build/server/Dockerfile . + @docker build --build-arg siteurl=${SITEURL} -t offen/proxy:${DOCKER_IMAGE_TAG} -f build/proxy/Dockerfile . secret: @docker-compose run server make secret diff --git a/README.md b/README.md index c3f77e2..dcd07c4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This repository contains all source code needed to build and run __offen__, both --- -Development of __offen__ has just started, so instructions are rare and things will stay highly volatile for quite some while. Also __do not use the software in its current state__ as it is still missing crucial pieces in protecting the data end to end. +Development of __offen__ has just started, so instructions are rare and things will stay highly volatile for quite some while. Guidelines for running and developing the Software will be added when it makes sense to do so. diff --git a/build/proxy/Dockerfile b/build/proxy/Dockerfile index a1f820d..c08e576 100644 --- a/build/proxy/Dockerfile +++ b/build/proxy/Dockerfile @@ -43,6 +43,9 @@ RUN npm run build FROM nikolaik/python-nodejs:python3.6-nodejs10 as homepage +ARG siteurl +ENV SITEURL=$siteurl + COPY ./homepage /code/homepage COPY ./styles /code/styles diff --git a/homepage/publishconf.py b/homepage/publishconf.py index b28fdae..16e9042 100644 --- a/homepage/publishconf.py +++ b/homepage/publishconf.py @@ -11,7 +11,7 @@ sys.path.append(os.curdir) from pelicanconf import * # If your site is available via HTTPS, make sure SITEURL begins with https:// -SITEURL = os.environ.get('PELICAN_SITEURL', 'https://www.offen.dev') +SITEURL = os.environ.get('SITEURL', 'https://www.offen.dev') # RELATIVE_URLS = True FEED_ALL_ATOM = 'feeds/all.atom.xml'