cache abstraction

angular.json

@@ -138,6 +138,33 @@
                     "style": "scss"
                 }
             }
+        },
+        "red-cache": {
+            "projectType": "library",
+            "root": "libs/red-cache",
+            "sourceRoot": "libs/red-cache/src",
+            "prefix": "redaction",
+            "architect": {
+                "lint": {
+                    "builder": "@angular-devkit/build-angular:tslint",
+                    "options": {
+                        "tsConfig": ["libs/red-cache/tsconfig.lib.json", "libs/red-cache/tsconfig.spec.json"],
+                        "exclude": ["**/node_modules/**", "!libs/red-cache/**/*"]
+                    }
+                },
+                "test": {
+                    "builder": "@nrwl/jest:jest",
+                    "options": {
+                        "jestConfig": "libs/red-cache/jest.config.js",
+                        "passWithNoTests": true
+                    }
+                }
+            },
+            "schematics": {
+                "@schematics/angular:component": {
+                    "style": "scss"
+                }
+            }
         }
     },
     "cli": {

apps/red-ui/ngsw-config.json

@@ -17,16 +17,5 @@
                 "files": ["/assets/**", "/*.(eot|svg|cur|jpg|png|webp|gif|otf|ttf|woff|woff2|ani)"]
             }
         }
-    ],
-    "dataGroups": [
-        {
-            "name": "file-downloads",
-            "urls": ["/download/original/**"],
-            "cacheConfig": {
-                "maxAge": "7d",
-                "maxSize": 1000,
-                "strategy": "performance"
-            }
-        }
     ]
 }

apps/red-ui/src/app/app.module.ts

@@ -77,6 +77,7 @@ import { TypeAnnotationIconComponent } from './components/type-annotation-icon/t
 import { TypeFilterComponent } from './components/type-filter/type-filter.component';
 import { DictionaryAnnotationIconComponent } from './components/dictionary-annotation-icon/dictionary-annotation-icon.component';
 import { BulkActionsComponent } from './screens/project-overview-screen/bulk-actions/bulk-actions.component';
+import { HttpCacheInterceptor } from '@redaction/red-cache';
 
 export function HttpLoaderFactory(httpClient: HttpClient) {
     return new TranslateHttpLoader(httpClient, '/assets/i18n/', '.json');
@@ -220,6 +221,11 @@ export function HttpLoaderFactory(httpClient: HttpClient) {
             multi: true,
             useClass: ApiPathInterceptorService
         },
+        {
+            provide: HTTP_INTERCEPTORS,
+            multi: true,
+            useClass: HttpCacheInterceptor
+        },
         {
             provide: APP_INITIALIZER,
             multi: true,

docker/common/nginx/nginx.conf.template

@@ -1,5 +1,5 @@
 server {
-  listen 8080;
+  listen 8080 http2;
   proxy_hide_header WWW-Authenticate;
   root /usr/share/nginx/html;
 

libs/red-cache/README.md

@@ -0,0 +1,7 @@
+# red-cache
+
+This library was generated with [Nx](https://nx.dev).
+
+## Running unit tests
+
+Run `nx test red-cache` to execute the unit tests.

libs/red-cache/jest.config.js

@@ -0,0 +1,18 @@
+module.exports = {
+    name: 'red-cache',
+    preset: '../../jest.config.js',
+    setupFilesAfterEnv: ['<rootDir>/src/test-setup.ts'],
+    globals: {
+        'ts-jest': {
+            tsConfig: '<rootDir>/tsconfig.spec.json',
+            stringifyContentPathRegex: '\\.(html|svg)$',
+            astTransformers: ['jest-preset-angular/build/InlineFilesTransformer', 'jest-preset-angular/build/StripStylesTransformer']
+        }
+    },
+    coverageDirectory: '../../coverage/libs/red-cache',
+    snapshotSerializers: [
+        'jest-preset-angular/build/AngularNoNgAttributesSnapshotSerializer.js',
+        'jest-preset-angular/build/AngularSnapshotSerializer.js',
+        'jest-preset-angular/build/HTMLCommentSerializer.js'
+    ]
+};

libs/red-cache/src/index.ts

@@ -0,0 +1,5 @@
+/*
+ * Public API Surface of communication-lib
+ */
+
+export * from './lib/index';

libs/red-cache/src/lib/caches/cache-api.service.ts

@@ -0,0 +1,279 @@
+import { Injectable } from '@angular/core';
+import { DYNAMIC_CACHES, APP_LEVEL_CACHE } from './cache-utils';
+import { HttpEvent, HttpHeaders, HttpRequest, HttpResponse } from '@angular/common/http';
+import { from, Observable, throwError } from 'rxjs';
+import { map, mergeMap } from 'rxjs/operators';
+
+@Injectable({
+    providedIn: 'root'
+})
+export class CacheApiService {
+    constructor() {
+        this.checkCachesExpiration();
+    }
+
+    cacheRequest(request: HttpRequest<any>, httpResponse: HttpResponse<any>): Promise<any> {
+        if (httpResponse.status < 300 && httpResponse.status >= 200) {
+            const url = this._buildUrl(request);
+            for (const dynCache of DYNAMIC_CACHES) {
+                for (const cacheUrl of dynCache.urls) {
+                    if (url.indexOf(cacheUrl) >= 0) {
+                        return caches.open(dynCache.name).then((cache) => {
+                            return this._handleFetchResponse(httpResponse, dynCache, cache, url);
+                        });
+                    }
+                }
+            }
+        }
+    }
+
+    getCachedRequest(request: HttpRequest<any>): Observable<HttpEvent<any>> {
+        const url = this._buildUrl(request);
+
+        return from(caches.match(url)).pipe(
+            mergeMap((response) => {
+                if (response) {
+                    const expires = response.headers.get('_expires');
+                    if (expires) {
+                        // if not expired, return, else override
+                        if (parseInt(expires, 10) > new Date().getTime()) {
+                            // console.log('[CACHE-API] Returning from cache: ', url);
+                            return this._toHttpResponse(response);
+                        } else {
+                            // console.log('[CACHE-API] cache expired: ', url);
+                        }
+                    } else {
+                        // console.log('[CACHE-API] Returning from cache: ', url);
+                        return this._toHttpResponse(response);
+                    }
+                }
+                return throwError(new Error('Request not Cached'));
+            })
+        );
+    }
+
+    cacheValue(name: string, valueReference: any, ttl = 3600): Promise<any> {
+        if (this.cachesAvailable) {
+            return caches.open(APP_LEVEL_CACHE).then((cache) => {
+                const string = JSON.stringify(valueReference);
+                const expires = new Date().getTime() + ttl * 1000;
+                const response = new Response(string, {
+                    headers: {
+                        _expires: `${expires}`
+                    }
+                });
+                const request = new Request(name);
+                // console.log('should cache', valueReference, string, response);
+                return cache.put(request, response);
+            });
+        } else {
+            return Promise.resolve();
+        }
+    }
+
+    removeCache(cacheId: string): Promise<any> {
+        if (this.cachesAvailable) {
+            return caches.open(APP_LEVEL_CACHE).then((cache) => {
+                return cache.delete(cacheId);
+            });
+        } else {
+            return Promise.resolve(undefined);
+        }
+    }
+
+    deleteDynamicCache(cacheName: string): Promise<any> {
+        if (this.cachesAvailable && DYNAMIC_CACHES.some((cache) => cache.name === cacheName)) {
+            return caches.delete(cacheName);
+        } else {
+            return Promise.resolve(undefined);
+        }
+    }
+
+    deleteDynamicCacheEntry(cacheName: string, cacheEntry: string): Promise<any> {
+        if (this.cachesAvailable && DYNAMIC_CACHES.some((cache) => cache.name === cacheName)) {
+            return caches.open(cacheName).then((cache) => {
+                return cache.delete(cacheEntry);
+            });
+        } else {
+            return Promise.resolve(undefined);
+        }
+    }
+
+    getCachedValue(name: string): Promise<any> {
+        if (this.cachesAvailable) {
+            return caches.open(APP_LEVEL_CACHE).then((cache) => {
+                return cache.match(name).then((result) => {
+                    if (result) {
+                        const expires = result.headers.get('_expires');
+                        try {
+                            if (parseInt(expires, 10) > new Date().getTime()) {
+                                return result.json();
+                            }
+                        } catch (e) {}
+                    }
+                    return undefined;
+                });
+            });
+        } else {
+            return Promise.resolve(undefined);
+        }
+    }
+
+    _buildUrl(request: HttpRequest<any>) {
+        let url;
+        if (request.method === 'GET') {
+            url = request.urlWithParams;
+        }
+        if (request.method === 'POST') {
+            const body = request.body;
+            let hash;
+            if (Array.isArray(body)) {
+                hash = JSON.stringify(body.sort());
+            } else {
+                hash = JSON.stringify(body);
+            }
+            const separator = request.urlWithParams.indexOf('?') > 0 ? '&' : '?';
+            url = request.urlWithParams + separator + btoa(hash);
+        }
+        return url;
+    }
+
+    isCachable(event: HttpRequest<any>) {
+        // only do shit for post and get
+
+        if (this.cachesAvailable && (event.method === 'GET' || event.method === 'POST')) {
+            // quick check if it has the potential of caching
+            const preliminaryUrl = event.url;
+            let tryCache = false;
+
+            for (const cache of DYNAMIC_CACHES) {
+                if (cache.methods.indexOf(event.method) >= 0) {
+                    for (const url of cache.urls) {
+                        if (preliminaryUrl.indexOf(url) >= 0) {
+                            tryCache = true;
+                            break;
+                        }
+                    }
+                }
+            }
+
+            return tryCache;
+        }
+        return false;
+    }
+
+    private _toHttpResponse(response: Response): Observable<HttpResponse<any>> {
+        response = response.clone();
+        const contentType = response.headers.get('content-type');
+
+        let obs: Observable<any>;
+        if (contentType) {
+            if (contentType.toLowerCase().indexOf('application/json') >= 0) {
+                obs = from(response.json());
+            }
+            if (contentType.toLowerCase().indexOf('text/') >= 0) {
+                obs = from(response.text());
+            }
+            if (contentType.toLowerCase().indexOf('image/') >= 0) {
+                if (contentType.toLowerCase().indexOf('image/svg') >= 0) {
+                    obs = from(response.text());
+                } else {
+                    obs = from(response.blob());
+                }
+            }
+            if (contentType.toLowerCase().indexOf('application/pdf') >= 0) {
+                obs = from(response.blob());
+            }
+        }
+        // console.log('[CACHE-API] content type', contentType, response.url);
+        return obs.pipe(
+            map((body) => {
+                // console.log('[CACHE-API] BODY', body);
+                return new HttpResponse({
+                    body,
+                    status: response.status,
+                    statusText: response.statusText,
+                    url: response.url,
+                    headers: this._toHttpHeaders(response.headers)
+                });
+            })
+        );
+    }
+
+    private _handleFetchResponse(httpResponse: HttpResponse<any>, dynCache, cache, url) {
+        const expires = new Date().getTime() + dynCache.maxAge * 1000;
+
+        const cachedResponseFields = {
+            status: httpResponse.status,
+            statusText: httpResponse.statusText,
+            headers: {
+                _expires: undefined
+            }
+        };
+
+        httpResponse.headers.keys().forEach((key) => {
+            cachedResponseFields.headers[key] = httpResponse.headers.get(key);
+        });
+        cachedResponseFields.headers._expires = expires;
+
+        let body;
+        if (cachedResponseFields.headers['content-type'].indexOf('application/json') >= 0) {
+            body = JSON.stringify(httpResponse.body);
+        } else {
+            body = httpResponse.body;
+        }
+        cache.put(url, new Response(body, cachedResponseFields));
+    }
+
+    checkCachesExpiration() {
+        if (this.cachesAvailable) {
+            const now = new Date().getTime();
+            for (const dynCache of DYNAMIC_CACHES) {
+                this._handleCacheExpiration(dynCache, now);
+            }
+        }
+    }
+
+    private _handleCacheExpiration(dynCache, now) {
+        caches.open(dynCache.name).then((cache) => {
+            cache.keys().then((keys) => {
+                // removed expired;
+                for (let i = 0; i < keys.length; i++) {
+                    const key = keys[i];
+                    // console.log('[CACHE-API] checking cache key: ', key);
+                    cache.match(key).then((response) => {
+                        const expires = response.headers.get('_expires');
+                        try {
+                            if (parseInt(expires, 10) < now) {
+                                console.log('remove cache', key);
+                                cache.delete(key);
+                            }
+                        } catch (e) {}
+                    });
+                }
+            });
+            cache.keys().then((keys) => {
+                if (keys.length > dynCache.maxSize) {
+                    const keysToRemove = keys.slice(0, keys.length - dynCache.maxSize);
+                    // console.log('[CACHE-API] cache to large - removing keys: ', keysToRemove);
+                    for (let i = 0; i < keysToRemove.length; i++) {
+                        const key = keys[i];
+                        cache.delete(key);
+                    }
+                }
+            });
+        });
+    }
+
+    get cachesAvailable(): boolean {
+        return !!window.caches;
+    }
+
+    private _toHttpHeaders(headers: Headers): HttpHeaders {
+        let httpHeaders = new HttpHeaders();
+        headers.forEach((value, key) => {
+            httpHeaders = httpHeaders.append(key, value);
+        });
+        return httpHeaders;
+    }
+}

libs/red-cache/src/lib/caches/cache-utils.ts

@@ -0,0 +1,39 @@
+export const APP_LEVEL_CACHE = 'app-level-cache';
+
+export const DYNAMIC_CACHES = [
+    {
+        urls: ['/assets/'],
+        name: 'cached-assets',
+        maxAge: 24 * 3600,
+        maxSize: 1000,
+        methods: ['GET']
+    },
+
+    {
+        urls: ['/download/original'],
+        name: 'files',
+        maxAge: 3600 * 24 * 7,
+        maxSize: 1000,
+        clearOnLogout: true,
+        methods: ['GET']
+    }
+];
+
+export async function wipeCaches(logoutDependant: boolean = false) {
+    await caches.delete(APP_LEVEL_CACHE);
+    for (const cache of DYNAMIC_CACHES) {
+        if (!logoutDependant) {
+            caches.delete(cache.name);
+        }
+        if (logoutDependant && cache.clearOnLogout) {
+            caches.delete(cache.name);
+        }
+    }
+}
+
+export async function wipeCacheEntry(cacheName: string, entry: string) {
+    caches.open(cacheName).then((cache) => {
+        console.log('delete:', entry);
+        cache.delete(entry, { ignoreSearch: false });
+    });
+}

libs/red-cache/src/lib/caches/cacheable.ts

@@ -0,0 +1,5 @@
+export interface Cacheable {
+    restoreFromCache(): Promise<any>;
+
+    storeToCache(): Promise<any>;
+}

libs/red-cache/src/lib/caches/http-cache-interceptor.ts

@@ -0,0 +1,43 @@
+import { HttpEvent, HttpHandler, HttpInterceptor, HttpRequest, HttpResponse } from '@angular/common/http';
+import { Injectable } from '@angular/core';
+import { Observable } from 'rxjs';
+import { CacheApiService } from './cache-api.service';
+import { catchError, tap } from 'rxjs/operators';
+
+@Injectable()
+export class HttpCacheInterceptor implements HttpInterceptor {
+    constructor(private cacheApiService: CacheApiService) {}
+
+    intercept(req: HttpRequest<any>, next: HttpHandler): Observable<HttpEvent<any>> {
+        // continue if not cachable.
+        if (!this.cacheApiService.isCachable(req)) {
+            return next.handle(req);
+        }
+
+        return this.cacheApiService.getCachedRequest(req).pipe(
+            tap((ok) => {
+                // console.log('[CACHE-API] got from cache', ok);
+            }),
+            catchError((cacheError) => {
+                // console.log("[CACHE-API] Cache fetch error", cacheError, req.url);
+                return this.sendRequest(req, next);
+            })
+        );
+    }
+
+    /**
+     * Get server response observable by sending request to `next()`.
+     * Will add the response to the cache on the way out.
+     */
+    sendRequest(request: HttpRequest<any>, next: HttpHandler): Observable<HttpEvent<any>> {
+        // console.log('[CACHE-API] request', request.url);
+        return next.handle(request).pipe(
+            tap((event) => {
+                // There may be other events besides the response.
+                if (event instanceof HttpResponse) {
+                    this.cacheApiService.cacheRequest(request, event);
+                }
+            })
+        );
+    }
+}

libs/red-cache/src/lib/caches/id-to-object-cache.register.service.ts

@@ -0,0 +1,11 @@
+import { IdToObjectListCacheStoreService } from './id-to-object-list-cache-store.service';
+import { Injectable } from '@angular/core';
+
+@Injectable({
+    providedIn: 'root'
+})
+export class IdToObjectCacheRegisterService {
+    constructor(private idToObjectListCacheStoreService: IdToObjectListCacheStoreService) {}
+
+    registerCaches() {}
+}

libs/red-cache/src/lib/caches/id-to-object-list-cache-store.service.ts

@@ -0,0 +1,94 @@
+import { Injectable } from '@angular/core';
+import { from, Observable, of } from 'rxjs';
+import { CacheApiService } from './cache-api.service';
+import { catchError, map, mergeMap } from 'rxjs/operators';
+
+export interface IdToObject {
+    id: string;
+    object: any;
+}
+
+export interface RegisteredCache {
+    name: string;
+    keyConversionFunction: (id: string, input: any) => string;
+    cacheFunction: (ids: string[], ...params: any) => Observable<{ [key: string]: any }>;
+}
+
+@Injectable({
+    providedIn: 'root'
+})
+export class IdToObjectListCacheStoreService {
+    private cachesList: { [key: string]: RegisteredCache } = {};
+
+    constructor(private cacheApiService: CacheApiService) {}
+
+    public registerCache(
+        name: string,
+        cacheFunction: (ids: string[], ...params: any) => Observable<{ [key: string]: any }>,
+        keyConversionFunction: (id: string, ...params: any) => string = (id, params) => id
+    ) {
+        this.cachesList[name] = {
+            name,
+            keyConversionFunction,
+            cacheFunction
+        };
+    }
+
+    public invokeCache(name: string, ids: string[], ...params: any): Observable<{ [key: string]: any }> {
+        const promises = [];
+        const cache = this.cachesList[name];
+        ids.map((id) => this._toUrl(name, cache.keyConversionFunction(id, params))).forEach((url) => {
+            promises.push(this.cacheApiService.getCachedValue(url));
+        });
+        return from(Promise.all(promises))
+            .pipe(catchError((error) => of([])))
+            .pipe(
+                mergeMap((resolvedValues: IdToObject[]) => {
+                    const partialResult = {};
+                    resolvedValues
+                        .filter((v) => !!v)
+                        .forEach((foundValue) => {
+                            partialResult[foundValue.id] = foundValue.object;
+                        });
+
+                    const existingIds = Object.keys(partialResult);
+                    const requestIds = ids.filter((el) => {
+                        return !existingIds.includes(el);
+                    });
+
+                    if (requestIds.length > 0) {
+                        return cache.cacheFunction(requestIds, params).pipe(
+                            map((data) => {
+                                // new items
+                                for (const key of Object.keys(data)) {
+                                    const idToObject = {
+                                        id: key,
+                                        object: data[key]
+                                    };
+                                    // cache each new result
+                                    this.cacheApiService.cacheValue(this._toUrl(name, cache.keyConversionFunction(key, params)), idToObject);
+                                }
+                                // add existing results to final result
+                                for (const existingKey of Object.keys(partialResult)) {
+                                    data[existingKey] = partialResult[existingKey];
+                                }
+                                return data;
+                            })
+                        );
+                    } else {
+                        return of(partialResult);
+                    }
+                })
+            );
+    }
+
+    public expireCache(name: string, id: string, ...params: any) {
+        const cache = this.cachesList[name];
+        const cacheUrl = this._toUrl(name, cache.keyConversionFunction(id, params));
+        this.cacheApiService.removeCache(cacheUrl);
+    }
+
+    private _toUrl(cacheName: string, id: string) {
+        return `/${cacheName}/${id}`;
+    }
+}

libs/red-cache/src/lib/caches/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2010 - 2030 by ACI Worldwide Inc.
+ * All rights reserved.
+ *
+ * This software is the confidential and proprietary information
+ * of ACI Worldwide Inc ("Confidential Information").  You shall
+ * not disclose such Confidential Information and shall use it
+ * only in accordance with the terms of the license agreement
+ * you entered with ACI Worldwide Inc.
+ *
+ */
+
+export * from './cache-api.service';
+export * from './cache-utils';
+export * from './cacheable';
+export * from './http-cache-interceptor';
+export * from './id-to-object-cache.register.service';
+export * from './id-to-object-list-cache-store.service';

libs/red-cache/src/lib/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2010 - 2030 by ACI Worldwide Inc.
+ * All rights reserved.
+ *
+ * This software is the confidential and proprietary information
+ * of ACI Worldwide Inc ("Confidential Information").  You shall
+ * not disclose such Confidential Information and shall use it
+ * only in accordance with the terms of the license agreement
+ * you entered with ACI Worldwide Inc.
+ *
+ */
+
+export * from './caches/index';

libs/red-cache/src/test.ts

@@ -0,0 +1,16 @@
+// This file is required by karma.conf.js and loads recursively all the .spec and framework files
+
+import 'core-js/es7/reflect';
+import 'zone.js/dist/zone';
+import 'zone.js/dist/zone-testing';
+import { getTestBed } from '@angular/core/testing';
+import { BrowserDynamicTestingModule, platformBrowserDynamicTesting } from '@angular/platform-browser-dynamic/testing';
+
+declare const require: any;
+
+// First, initialize the Angular testing environment.
+getTestBed().initTestEnvironment(BrowserDynamicTestingModule, platformBrowserDynamicTesting());
+// Then we find all the tests.
+const context = require.context('./', true, /\.spec\.ts$/);
+// And load the modules.
+context.keys().map(context);

libs/red-cache/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.lib.json"
+        },
+        {
+            "path": "./tsconfig.spec.json"
+        }
+    ]
+}

libs/red-cache/tsconfig.lib.json

@@ -0,0 +1,19 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../dist/out-tsc",
+        "target": "es2015",
+        "declaration": true,
+        "declarationMap": true,
+        "inlineSources": true,
+        "types": [],
+        "lib": ["dom", "es2018"]
+    },
+    "angularCompilerOptions": {
+        "skipTemplateCodegen": true,
+        "strictMetadataEmit": true,
+        "enableResourceInlining": true
+    },
+    "exclude": ["src/test-setup.ts", "**/*.spec.ts"],
+    "include": ["**/*.ts"]
+}

libs/red-cache/tsconfig.spec.json

@@ -0,0 +1,10 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../dist/out-tsc",
+        "module": "commonjs",
+        "types": ["jest", "node"]
+    },
+    "files": ["src/test-setup.ts"],
+    "include": ["**/*.spec.ts", "**/*.d.ts"]
+}

libs/red-cache/tslint.json

@@ -0,0 +1,10 @@
+{
+    "extends": "../../tslint.json",
+    "rules": {
+        "directive-selector": [true, "attribute", "redaction", "camelCase"],
+        "component-selector": [true, "element", "redaction", "kebab-case"]
+    },
+    "linterOptions": {
+        "exclude": ["!**/*"]
+    }
+}

nx.json

@@ -27,6 +27,9 @@
         },
         "red-ui-http": {
             "tags": []
+        },
+        "red-cache": {
+            "tags": []
         }
     }
 }

tsconfig.base.json

@@ -17,7 +17,8 @@
         "skipDefaultLibCheck": true,
         "baseUrl": ".",
         "paths": {
-            "@redaction/red-ui-http": ["libs/red-ui-http/src/index.ts"]
+            "@redaction/red-ui-http": ["libs/red-ui-http/src/index.ts"],
+            "@redaction/red-cache": ["libs/red-cache/src/index.ts"]
         }
     },
     "exclude": ["node_modules", "tmp"]

tsconfig.json

@@ -1,3 +0,0 @@
-{
-    "extends": "./tsconfig.base.json"
-}