Upgrade chromium and Overview

Now we add overview-convert-pdf and disable OCR. See `README.md` for
upgrade instructions.

Author: adamhooper

README.md

@@ -198,3 +198,35 @@ always be both; don't let Capybara's confusing design confuse you.
 http://www.rubydoc.info/github/jnicklas/capybara/Capybara/Session
 and other spots online. One detail sets us apart: we do _not_ use
 `Capybara.default_max_wait_time` because it is terrible.
+
+Changelog and Update Instructions
+=================================
+
+**2.0.0**, released 2018-04-10
+
+Changes:
+
+* Upgraded to Chromedriver 2.37 and Chromium 64
+* Switched to debian-slim for faster pull
+* Upgraded Overview
+
+To upgrade:
+
+* Change `OVERVIEW_INTEGRATION_TESTER_VERSION` to `2.0.0` and update
+  `OVERVIEW_VERSION` to at least `ba89c4502319969f3bd9248150e8608c085d7c7e`
+* Add `OV_APPLICATION_SECRET` environment variable to `overview-web` with
+  any String value
+* Add `overview-convert-pdf` instance, if your tests use any PDF imports
+* Overwrite `helpers/session_helpers/document_set_helper.rb`. (It now disables
+  OCR when uploading PDFs: that lets you omit `overview-convert-pdfocr` from
+  `docker-compose.yml` and speed up tests.)
+
+Releasing This Framework
+========================
+
+1. Fix a bug or add a feature, presumably by writing tests in `test/` and/or
+   editing code in `skeleton/`.
+2. Pick a new version number, `X.Y.Z` (consulting `VERSION` and semver).
+3. `docker build docker -t overview/overview-integration-tester:X.Y.Z`
+4. `./test/test.sh`
+5. `./release X.Y.Z`

docker/Dockerfile

@@ -5,13 +5,13 @@
 #
 # Switch to alpine when it has chromium 63. Debian (Chrome 63) makes the build
 # super-slow.
-FROM debian:stretch
+FROM debian:stretch-slim
 
 RUN set -x \
       && echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/force-unsafe-io \
       && echo "Acquire::http {No-Cache=True;};" > /etc/apt/apt.conf.d/no-cache \
       && apt-get update \
-      && apt-get install -y \
+      && apt-get install -y --no-install-recommends \
         build-essential \
         ca-certificates \
         chromium \
@@ -24,8 +24,9 @@ RUN set -x \
         zlib1g-dev \
         tigervnc-standalone-server \
         xvfb \
+      && apt-get clean -y && rm -rf /var/cache/debconf/* /var/lib/apt/lists/* /var/log/* /tmp/* /var/tmp/* \
       && gem install bundler \
-      && (cd /tmp && curl "http://chromedriver.storage.googleapis.com/2.%3Cspan%20class="x x-first x-last">35/chromedriver_linux64.zip" -o chromedriver_linux64.zip && unzip chromedriver_linux64.zip && rm -f chromedriver_linux64.zip && mv chromedriver /usr/bin/)
+      && (cd /tmp && curl "http://chromedriver.storage.googleapis.com/2.%3Cspan%20class="x x-first x-last">37/chromedriver_linux64.zip" -o chromedriver_linux64.zip && unzip chromedriver_linux64.zip && rm -f chromedriver_linux64.zip && mv chromedriver /usr/bin/)
 
 # Expose remote debugging port. Browse to http://docker-container-ip:9222
 # to see what's going on inside Chromium.

skeleton/config

@@ -20,15 +20,15 @@ DEVELOPMENT_PROJECT_NAME=overviewserver
 
 # Docker container version. We use semantic versioning; only the major version
 # matters, but you can be paranoid and add minor and patch versions, too.
-OVERVIEW_INTEGRATION_TESTER_VERSION=1.1.0
+OVERVIEW_INTEGRATION_TESTER_VERSION=2.0.0
 
 # Version of Overview to test against in `run-in-docker-compose`. See
 # https://hub.docker.com/r/overview/overview-database/tags/ for options.
 #
 # Do not use `latest`. Overview changes in ways that might break your tests.
 # (For instance, a button might be renamed.) You should frequently update your
 # tests to the latest version of Overview.
-OVERVIEW_VERSION=b7653f60b6fdcbf45237110a6d18748a11d4e05e
+OVERVIEW_VERSION=ba89c4502319969f3bd9248150e8608c085d7c7e
 
 # Stall `run` and `run-in-docker-compose` on startup until these URLs are
 # available.

skeleton/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     image: overview/overview-web:${OVERVIEW_VERSION}
     networks: [ 'default', 'database', 'redis', 'worker' ]
     depends_on: [ 'overview-database', 'overview-redis', 'overview-worker' ]
+    environment:
+      OV_APPLICATION_SECRET: 'this-is-a-secret-you-must-set-in-production'
     volumes:
       - blob-storage:/var/lib/overview/blob-storage
 
@@ -29,14 +31,24 @@ services:
     networks: [ 'database' ]
     depends_on: [ 'overview-database' ]
 
+  # This integration-test environment only supports PDFs without OCR. To support
+  # more input formats, add more converters to docker-compose. (See
+  # overview-server/docker-compose.yml and overview-server/.env for a full list
+  # of converters.)
+  overview-convert-pdf:
+    image: overview/overview-convert-pdf:0.0.3
+    networks: [ 'worker' ]
+    environment:
+      POLL_URL: 'http://overview-worker:9032/Pdf'
+
   # === Your project (e.g., your plugin): you probably _do_ want to edit this ===
   my-project:
     build: ..
 
   # === The integration-test runner ===
   integration-test:
     image: overview/overview-integration-tester:${OVERVIEW_INTEGRATION_TESTER_VERSION}
-    depends_on: [ 'overview-web', 'my-project' ]
+    depends_on: [ 'overview-web', 'overview-convert-pdf', 'my-project' ]
     environment:
       OVERVIEW_URL: http://overview-web
       WAIT_FOR_URLS: ${WAIT_FOR_URLS}

skeleton/helpers/session_helpers/document_set_helper.rb

@@ -12,10 +12,12 @@ def create_document_set_from_pdfs_in_folder(folder)
       for path in Dir.glob(File.join(folder, '*.*'))
         attach_file('file', path)
       end
+
       click_on('Done adding files')
       # Wait for focus: that's when the dialog is open
       wait_for_javascript_to_return_true('document.querySelector("#import-options-name") === document.activeElement', wait: WAIT_FAST)
       fill_in('Document set name', with: folder)
+      choose('Assume documents are already text (faster import)')
       click_on('Import documents')
       assert_selector('body.document-set-show', wait: WAIT_SLOW) # wait for import to complete
       assert_selector('#document-list:not(.loading) li.document', wait: WAIT_LOAD) # wait for document list to load

test/files-atop-framework/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     image: overview/overview-web:${OVERVIEW_VERSION}
     networks: [ 'default', 'database', 'redis', 'worker' ]
     depends_on: [ 'overview-database', 'overview-redis', 'overview-worker' ]
+    environment:
+      OV_APPLICATION_SECRET: 'this-is-a-secret-you-must-set-in-production'
     volumes:
       - blob-storage:/var/lib/overview/blob-storage
 
@@ -29,10 +31,20 @@ services:
     networks: [ 'database' ]
     depends_on: [ 'overview-database' ]
 
+  # This integration-test environment only supports PDFs without OCR. To support
+  # more input formats, add more converters to docker-compose. (See
+  # overview-server/docker-compose.yml and overview-server/.env for a full list
+  # of converters.)
+  overview-convert-pdf:
+    image: overview/overview-convert-pdf:0.0.3
+    networks: [ 'worker' ]
+    environment:
+      POLL_URL: 'http://overview-worker:9032/Pdf'
+
   # === The integration-test runner ===
   integration-test:
     image: overview/overview-integration-tester:${OVERVIEW_INTEGRATION_TESTER_VERSION}
-    depends_on: [ 'overview-web' ]
+    depends_on: [ 'overview-web', 'overview-convert-pdf' ]
     environment:
       OVERVIEW_URL: http://overview-web
       WAIT_FOR_URLS: ${WAIT_FOR_URLS}