Rewrite Docker image to function with current HEAD and upstream wego

.gitignore

@@ -2,6 +2,7 @@ ve/
 share/static/fonts/
 *.pyc
 data/
+cache/
 log/
 .idea/
 *.swp

Dockerfile

@@ -1,71 +1,68 @@
 # Build stage
-FROM golang:1-alpine as builder
+FROM golang:1-buster as builder
 
 WORKDIR /app
 
 COPY ./share/we-lang/we-lang.go /app
 COPY ./share/we-lang/go.mod /app
 
-RUN apk add --no-cache git
+RUN apt update && apt install -y git
 
 
 RUN go get -u github.com/mattn/go-colorable && \
     go get -u github.com/klauspost/lctime && \
     go get -u github.com/mattn/go-runewidth && \
-    CGO_ENABLED=0 go build /app/we-lang.go
-# Results in /app/we-lang
+    go get -u github.com/schachmat/wego && \
+    CGO_ENABLED=0 go build /app/we-lang.go && \
+    cp $GOPATH/bin/wego /app/wego
+# Results in /app/we-lang & /app/wego
 
 
-FROM alpine:3
+FROM python:3.9-slim-buster
 
 WORKDIR /app
 
-COPY ./requirements.txt /app
-
-ENV LLVM_CONFIG=/usr/bin/llvm10-config
-
-RUN apk add --no-cache --virtual .build \
-    autoconf \
-    automake \
-    g++ \
-    gcc \
-    jpeg-dev \
-    llvm11-dev\
-    make \
-    zlib-dev \
-    && apk add --no-cache \
-    python3 \
-    py3-pip \
-    py3-scipy \
-    py3-wheel \
-    py3-gevent \
-    zlib \
-    jpeg \
-    llvm11 \
-    libtool \
-    supervisor \
-    py3-numpy-dev \
-    python3-dev && \
-    mkdir -p /app/cache && \
-    mkdir -p /var/log/supervisor && \
-    mkdir -p /etc/supervisor/conf.d && \
-    chmod -R o+rw /var/log/supervisor && \
-    chmod -R o+rw /var/run && \
-    pip install -r requirements.txt --no-cache-dir && \
-    apk del --no-cache -r .build
+RUN mkdir -p cache data log \
+      /var/log/supervisor && \
+    chmod -R o+rw /var/log/supervisor /var/run cache log
 
 COPY --from=builder /app/we-lang /app/bin/we-lang
+COPY --from=builder /app/wego /app/bin/wego
 COPY ./bin /app/bin
 COPY ./lib /app/lib
 COPY ./share /app/share
 COPY share/docker/supervisord.conf /etc/supervisor/supervisord.conf
 
+# Get GeoLite2 & airports.dat
+ARG geolite_license_key
+RUN ( apt update && apt install -y wget && \
+  cd data/ && \
+  wget "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${geolite_license_key}&suffix=tar.gz" -O - | tar -xz --strip=1 --wildcards --no-anchored '*.mmdb' && \
+  wget "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat" -O airports.dat )
+
+
+COPY ./requirements.txt /app
+# Python build time dependencies
+RUN apt install -y \
+    build-essential \
+    autoconf \
+    libtool \
+    git \
+    # Runtime deps
+    gawk \
+    tzdata && \
+    pip install -r requirements.txt --no-cache-dir && \
+    apt remove -y \
+    build-essential \
+    autoconf \
+    libtool \
+    git && \
+    apt autoremove -y
+
 ENV WTTR_MYDIR="/app"
-ENV WTTR_GEOLITE="/app/GeoLite2-City.mmdb"
-ENV WTTR_WEGO="/app/bin/we-lang"
 ENV WTTR_LISTEN_HOST="0.0.0.0"
 ENV WTTR_LISTEN_PORT="8002"
 
 EXPOSE 8002
 
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
+CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]

README.md

@@ -617,13 +617,14 @@ because the MaxMind database is pretty good).
 
 * Install Docker
 * Build Docker Image
+   ```
+   docker build . --build-arg geolite_license_key=************
+   ```
 * These files should be mounted by the user at runtime:
 
 ```
 /root/.wegorc
 /root/.ip2location.key (optional)
-/app/airports.dat
-/app/GeoLite2-City.mmdb
 ```
 
 ### Get a WorldWeatherOnline key and configure wego

bin/srv.py

@@ -21,7 +21,7 @@
 from globals import TEMPLATES, STATIC, LISTEN_HOST, LISTEN_PORT
 # pylint: enable=wrong-import-position,wrong-import-order
 
-from view.v3 import v3_file
+# from view.v3 import v3_file
 
 MY_LOADER = jinja2.ChoiceLoader([
     APP.jinja_loader,
@@ -30,12 +30,12 @@
 
 APP.jinja_loader = MY_LOADER
 
-@APP.route('/v3/<string:location>')
-def send_v3(location):
-    filepath = v3_file(location)
-    if filepath.startswith("ERROR"):
-        return filepath.rstrip("\n") + "\n"
-    return send_file(filepath)
+# @APP.route('/v3/<string:location>')
+# def send_v3(location):
+#     filepath = v3_file(location)
+#     if filepath.startswith("ERROR"):
+#         return filepath.rstrip("\n") + "\n"
+#     return send_file(filepath)
 
 @APP.route('/files/<path:path>')
 def send_static(path):

lib/airports.py

@@ -1,9 +1,9 @@
 import csv
 
-AIRPORTS_DAT_FILE = '/home/igor/wttrin-geo/share/airports.dat'
+from globals import AIRPORTS_DAT
 
 def load_aiports_index():
-    file_ = open(AIRPORTS_DAT_FILE, "r")
+    file_ = open(AIRPORTS_DAT, "r")
     reader = csv.reader(file_)
     airport_index = {}
 

lib/globals.py

@@ -9,6 +9,7 @@
     WTTR_LISTEN_HOST
     WTTR_LISTEN_PORT
     WTTR_USER_AGENT
+    WTTR_AIRPORTS_DAT
 
 """
 from __future__ import print_function
@@ -18,17 +19,22 @@
 import re
 
 MYDIR = os.path.abspath(os.path.dirname(os.path.dirname('__file__')))
+_DATADIR = os.path.join(MYDIR, 'data')
+_LOGDIR = os.path.join(MYDIR, 'log')
 
 if "WTTR_GEOLITE" in os.environ:
     GEOLITE = os.environ["WTTR_GEOLITE"]
 else:
-    GEOLITE = os.path.join(MYDIR, 'data', "GeoLite2-City.mmdb")
+    GEOLITE = os.path.join(_DATADIR, "GeoLite2-City.mmdb")
 
-WEGO = os.environ.get("WTTR_WEGO", "/home/igor/go/bin/we-lang")
+if "WTTR_AIRPORTS_DAT" in os.environ:
+    AIRPORTS_DAT = os.environ["WTTR_AIRPORTS_DAT"]
+else:
+    AIRPORTS_DAT = os.path.join(_DATADIR, "airports.dat")
+
+WEGO = os.environ.get("WTTR_WEGO", os.path.join(MYDIR, 'bin', 'we-lang'))
 PYPHOON = "pyphoon-lolcat"
 
-_DATADIR = "/wttr.in"
-_LOGDIR = "/wttr.in/log"
 
 IP2LCACHE = os.path.join(_DATADIR, "cache/ip2l/")
 PNG_CACHE = os.path.join(_DATADIR, "cache/png")

lib/view/line.py

@@ -28,7 +28,6 @@
 from constants import WWO_CODE, WEATHER_SYMBOL, WEATHER_SYMBOL_WI_NIGHT, WEATHER_SYMBOL_WI_DAY, WIND_DIRECTION, WIND_DIRECTION_WI, WEATHER_SYMBOL_WIDTH_VTE, WEATHER_SYMBOL_PLAIN
 from weather_data import get_weather_data
 from . import v2
-from . import v3
 from . import prometheus
 
 PRECONFIGURED_FORMAT = {
@@ -386,7 +385,7 @@ def format_weather_data(query, parsed_query, data):
     if format_line[:2] == "v2":
         return v2.main(query, parsed_query, data)
     if format_line[:2] == "v3":
-        return v3.main(query, parsed_query, data)
+        raise NotImplementedError()
 
     current_condition = data['data']['current_condition'][0]
     current_condition['location'] = parsed_query["location"]

lib/view/wttr.py

@@ -8,6 +8,7 @@
 
 import sys
 import re
+import logging
 
 from gevent.subprocess import Popen, PIPE
 
@@ -22,6 +23,7 @@ def get_wetter(parsed_query):
     location = parsed_query['location']
     html = parsed_query['html_output']
     lang = parsed_query['lang']
+    logging.debug(f'get_wetter for {location}')
 
     location_not_found = False
     if location == NOT_FOUND_LOCATION:
@@ -30,8 +32,11 @@ def get_wetter(parsed_query):
     stderr = ""
     returncode = 0
     if not location_not_found:
+        logging.debug('Valid location, querying wego')
         stdout, stderr, returncode = _wego_wrapper(location, parsed_query)
 
+    if returncode != 0:
+        logging.debug(f'Wego failed with "{location}": {stderr}')
     if location_not_found or \
         (returncode != 0 \
             and ('Unable to find any matching weather'
@@ -77,7 +82,7 @@ def _wego_wrapper(location, parsed_query):
     else:
         location_name = parsed_query['override_location_name']
 
-    cmd = [WEGO, '--city=%s' % location]
+    cmd = [WEGO, '--location=%s' % location]
 
     if parsed_query.get('inverted_colors'):
         cmd += ['-inverse']
@@ -94,9 +99,7 @@ def _wego_wrapper(location, parsed_query):
     if parsed_query.get('use_imperial', False):
         cmd += ['-imperial']
 
-    if location_name:
-        cmd += ['-location_name', location_name]
-
+    logging.debug(f'wego cmd: "{cmd}"')
     proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
     stdout, stderr = proc.communicate()
     stdout = stdout.decode("utf-8")
@@ -158,12 +161,16 @@ def _htmlize(ansi_output, title, parsed_query):
     if not parsed_query.get('inverted_colors'):
         cmd += ["--bg=dark"]
 
+    logging.debug(f'Running ansi2html: "{cmd}"')
     proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
     stdout, stderr = proc.communicate(ansi_output.encode("utf-8"))
     stdout = stdout.decode("utf-8")
     stderr = stderr.decode("utf-8")
     if proc.returncode != 0:
+        logging.error(f'ansi2html failed: {stderr}')
         error(stdout + stderr)
+    else:
+        logging.debug(f'ansi2html did not fail: "{stderr}" and produced output: {stdout}')
 
     if parsed_query.get('inverted_colors'):
         stdout = stdout.replace(

lib/wttr_srv.py

@@ -57,6 +57,7 @@ def show_text_file(name, lang):
         text = text\
                 .replace('NUMBER_OF_LANGUAGES', str(len(SUPPORTED_LANGS)))\
                 .replace('SUPPORTED_LANGUAGES', ' '.join(SUPPORTED_LANGS))
+    logging.debug(f'Showing text for {name}')
     return text
 
 def _client_ip_address(request):
@@ -199,28 +200,34 @@ def _response(parsed_query, query, fast_mode=False):
     answer = cache.get(cache_signature)
 
     if parsed_query['orig_location'] in PLAIN_TEXT_PAGES:
+        logging.debug('Returning static data page')
         answer = show_text_file(parsed_query['orig_location'], parsed_query['lang'])
         if parsed_query['html_output']:
             answer = render_template('index.html', body=answer)
 
     if answer or fast_mode:
+        logging.debug('Returning cached result. Or failing on fast_mode.')
         return answer
 
     # at this point, we could not handle the query fast,
     # so we handle it with all available logic
     loc = (parsed_query['orig_location'] or "").lower()
+    logging.debug(f'No luck with cache for "{loc}", getting data for real.')
     if parsed_query.get("view"):
+        logging.debug(f'Querying wttr_line: {query}')
         if not parsed_query.get("location"):
             parsed_query["location"] = loc
-
         output = wttr_line(query, parsed_query)
     elif loc == 'moon' or loc.startswith('moon@'):
+        logging.debug(f'Querying moon')
         output = get_moon(parsed_query)
     else:
+        logging.debug(f'Querying wetter')
         output = get_wetter(parsed_query)
 
     if parsed_query.get('png_filename'):
         if parsed_query.get("view") != "v3":
+            logging.debug(f'Rendering results into a png')
             # originally it was just a usual function call,
             # but it was a blocking call, so it was moved
             # to separate threads:
@@ -233,14 +240,15 @@ def _response(parsed_query, query, fast_mode=False):
         if query.get('days', '3') != '0' \
             and not query.get('no-follow-line') \
             and ((parsed_query.get("view") or "v2")[:2] in ["v2", "v3"]):
+            logging.debug(f'Follow section not disabled, rendering schwag.')
             if parsed_query['html_output']:
                 output = add_buttons(output)
             else:
                 message = get_message('FOLLOW_ME', parsed_query['lang'])
                 if parsed_query.get('no-terminal', False):
                     message = remove_ansi(message)
                 output += '\n' + message + '\n'
-
+    logging.debug(f'Slow mode complete')
     return cache.store(cache_signature, output)
 
 def parse_request(location, request, query, fast_mode=False):
@@ -377,7 +385,9 @@ def _wrap_response(response_text, html_output, json_output, png_filename=None):
     try:
         if not response:
             parsed_query = parse_request(location, request, query)
+            logging.debug(f'Query: {parsed_query}')
             response = _response(parsed_query, query)
+            logging.debug(f'Result: {response}')
             #if not response or (isinstance(response, str) and not response.strip()):
             #    return RuntimeError("Empty answer")
 

share/docker/supervisord.conf

@@ -1,22 +1,16 @@
 [supervisord]
 nodaemon=true
-logfile=/var/log/supervisor/supervisord.log
 pidfile=/var/run/supervisord.pid
+redirect_stderr=true
 
 [program:srv]
 command=python3 /app/bin/srv.py
-stderr_logfile=/var/log/supervisor/srv-stderr.log
-stdout_logfile=/var/log/supervisor/srv-stdout.log
+redirect_stderr=true
 
 [program:proxy]
 command=python3 /app/bin/proxy.py
-stderr_logfile=/var/log/supervisor/proxy-stderr.log
-stdout_logfile=/var/log/supervisor/proxy-stdout.log
+redirect_stderr=true
 
 [program:geoproxy]
 command=python3 /app/bin/geo-proxy.py
-stderr_logfile=/var/log/supervisor/geoproxy-stderr.log
-stdout_logfile=/var/log/supervisor/geoproxy-stdout.log
-
-[include]
-files=/etc/supervisor/conf.d/*.conf
+redirect_stderr=true