diff --git a/.gitignore b/.gitignore index 5d3c7548..a5355fe2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ ve/ share/static/fonts/ *.pyc data/ +cache/ log/ .idea/ *.swp diff --git a/Dockerfile b/Dockerfile index 24f1a454..6b806214 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,71 +1,68 @@ # Build stage -FROM golang:1-alpine as builder +FROM golang:1-buster as builder WORKDIR /app COPY ./share/we-lang/we-lang.go /app COPY ./share/we-lang/go.mod /app -RUN apk add --no-cache git +RUN apt update && apt install -y git RUN go get -u github.com/mattn/go-colorable && \ go get -u github.com/klauspost/lctime && \ go get -u github.com/mattn/go-runewidth && \ - CGO_ENABLED=0 go build /app/we-lang.go -# Results in /app/we-lang + go get -u github.com/schachmat/wego && \ + CGO_ENABLED=0 go build /app/we-lang.go && \ + cp $GOPATH/bin/wego /app/wego +# Results in /app/we-lang & /app/wego -FROM alpine:3 +FROM python:3.9-slim-buster WORKDIR /app -COPY ./requirements.txt /app - -ENV LLVM_CONFIG=/usr/bin/llvm10-config - -RUN apk add --no-cache --virtual .build \ - autoconf \ - automake \ - g++ \ - gcc \ - jpeg-dev \ - llvm11-dev\ - make \ - zlib-dev \ - && apk add --no-cache \ - python3 \ - py3-pip \ - py3-scipy \ - py3-wheel \ - py3-gevent \ - zlib \ - jpeg \ - llvm11 \ - libtool \ - supervisor \ - py3-numpy-dev \ - python3-dev && \ - mkdir -p /app/cache && \ - mkdir -p /var/log/supervisor && \ - mkdir -p /etc/supervisor/conf.d && \ - chmod -R o+rw /var/log/supervisor && \ - chmod -R o+rw /var/run && \ - pip install -r requirements.txt --no-cache-dir && \ - apk del --no-cache -r .build +RUN mkdir -p cache data log \ + /var/log/supervisor && \ + chmod -R o+rw /var/log/supervisor /var/run cache log COPY --from=builder /app/we-lang /app/bin/we-lang +COPY --from=builder /app/wego /app/bin/wego COPY ./bin /app/bin COPY ./lib /app/lib COPY ./share /app/share COPY share/docker/supervisord.conf /etc/supervisor/supervisord.conf +# Get GeoLite2 & airports.dat +ARG geolite_license_key +RUN ( apt update && apt install -y wget && \ + cd data/ && \ + wget "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${geolite_license_key}&suffix=tar.gz" -O - | tar -xz --strip=1 --wildcards --no-anchored '*.mmdb' && \ + wget "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat" -O airports.dat ) + + +COPY ./requirements.txt /app +# Python build time dependencies +RUN apt install -y \ + build-essential \ + autoconf \ + libtool \ + git \ + # Runtime deps + gawk \ + tzdata && \ + pip install -r requirements.txt --no-cache-dir && \ + apt remove -y \ + build-essential \ + autoconf \ + libtool \ + git && \ + apt autoremove -y + ENV WTTR_MYDIR="/app" -ENV WTTR_GEOLITE="/app/GeoLite2-City.mmdb" -ENV WTTR_WEGO="/app/bin/we-lang" ENV WTTR_LISTEN_HOST="0.0.0.0" ENV WTTR_LISTEN_PORT="8002" EXPOSE 8002 -CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] +CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/README.md b/README.md index af3b349b..002f8f80 100644 --- a/README.md +++ b/README.md @@ -617,13 +617,14 @@ because the MaxMind database is pretty good). * Install Docker * Build Docker Image + ``` + docker build . --build-arg geolite_license_key=************ + ``` * These files should be mounted by the user at runtime: ``` /root/.wegorc /root/.ip2location.key (optional) -/app/airports.dat -/app/GeoLite2-City.mmdb ``` ### Get a WorldWeatherOnline key and configure wego diff --git a/bin/srv.py b/bin/srv.py index 83bccf45..e40524e7 100644 --- a/bin/srv.py +++ b/bin/srv.py @@ -21,7 +21,7 @@ from globals import TEMPLATES, STATIC, LISTEN_HOST, LISTEN_PORT # pylint: enable=wrong-import-position,wrong-import-order -from view.v3 import v3_file +# from view.v3 import v3_file MY_LOADER = jinja2.ChoiceLoader([ APP.jinja_loader, @@ -30,12 +30,12 @@ APP.jinja_loader = MY_LOADER -@APP.route('/v3/') -def send_v3(location): - filepath = v3_file(location) - if filepath.startswith("ERROR"): - return filepath.rstrip("\n") + "\n" - return send_file(filepath) +# @APP.route('/v3/') +# def send_v3(location): +# filepath = v3_file(location) +# if filepath.startswith("ERROR"): +# return filepath.rstrip("\n") + "\n" +# return send_file(filepath) @APP.route('/files/') def send_static(path): diff --git a/lib/airports.py b/lib/airports.py index e950ceb6..edf901df 100644 --- a/lib/airports.py +++ b/lib/airports.py @@ -1,9 +1,9 @@ import csv -AIRPORTS_DAT_FILE = '/home/igor/wttrin-geo/share/airports.dat' +from globals import AIRPORTS_DAT def load_aiports_index(): - file_ = open(AIRPORTS_DAT_FILE, "r") + file_ = open(AIRPORTS_DAT, "r") reader = csv.reader(file_) airport_index = {} diff --git a/lib/globals.py b/lib/globals.py index edc5e246..88d70be5 100644 --- a/lib/globals.py +++ b/lib/globals.py @@ -9,6 +9,7 @@ WTTR_LISTEN_HOST WTTR_LISTEN_PORT WTTR_USER_AGENT + WTTR_AIRPORTS_DAT """ from __future__ import print_function @@ -18,17 +19,22 @@ import re MYDIR = os.path.abspath(os.path.dirname(os.path.dirname('__file__'))) +_DATADIR = os.path.join(MYDIR, 'data') +_LOGDIR = os.path.join(MYDIR, 'log') if "WTTR_GEOLITE" in os.environ: GEOLITE = os.environ["WTTR_GEOLITE"] else: - GEOLITE = os.path.join(MYDIR, 'data', "GeoLite2-City.mmdb") + GEOLITE = os.path.join(_DATADIR, "GeoLite2-City.mmdb") -WEGO = os.environ.get("WTTR_WEGO", "/home/igor/go/bin/we-lang") +if "WTTR_AIRPORTS_DAT" in os.environ: + AIRPORTS_DAT = os.environ["WTTR_AIRPORTS_DAT"] +else: + AIRPORTS_DAT = os.path.join(_DATADIR, "airports.dat") + +WEGO = os.environ.get("WTTR_WEGO", os.path.join(MYDIR, 'bin', 'we-lang')) PYPHOON = "pyphoon-lolcat" -_DATADIR = "/wttr.in" -_LOGDIR = "/wttr.in/log" IP2LCACHE = os.path.join(_DATADIR, "cache/ip2l/") PNG_CACHE = os.path.join(_DATADIR, "cache/png") diff --git a/lib/view/line.py b/lib/view/line.py index 7bc81ef9..c6163558 100644 --- a/lib/view/line.py +++ b/lib/view/line.py @@ -28,7 +28,6 @@ from constants import WWO_CODE, WEATHER_SYMBOL, WEATHER_SYMBOL_WI_NIGHT, WEATHER_SYMBOL_WI_DAY, WIND_DIRECTION, WIND_DIRECTION_WI, WEATHER_SYMBOL_WIDTH_VTE, WEATHER_SYMBOL_PLAIN from weather_data import get_weather_data from . import v2 -from . import v3 from . import prometheus PRECONFIGURED_FORMAT = { @@ -386,7 +385,7 @@ def format_weather_data(query, parsed_query, data): if format_line[:2] == "v2": return v2.main(query, parsed_query, data) if format_line[:2] == "v3": - return v3.main(query, parsed_query, data) + raise NotImplementedError() current_condition = data['data']['current_condition'][0] current_condition['location'] = parsed_query["location"] diff --git a/lib/view/wttr.py b/lib/view/wttr.py index 1c0b2052..05e81178 100644 --- a/lib/view/wttr.py +++ b/lib/view/wttr.py @@ -8,6 +8,7 @@ import sys import re +import logging from gevent.subprocess import Popen, PIPE @@ -22,6 +23,7 @@ def get_wetter(parsed_query): location = parsed_query['location'] html = parsed_query['html_output'] lang = parsed_query['lang'] + logging.debug(f'get_wetter for {location}') location_not_found = False if location == NOT_FOUND_LOCATION: @@ -30,8 +32,11 @@ def get_wetter(parsed_query): stderr = "" returncode = 0 if not location_not_found: + logging.debug('Valid location, querying wego') stdout, stderr, returncode = _wego_wrapper(location, parsed_query) + if returncode != 0: + logging.debug(f'Wego failed with "{location}": {stderr}') if location_not_found or \ (returncode != 0 \ and ('Unable to find any matching weather' @@ -77,7 +82,7 @@ def _wego_wrapper(location, parsed_query): else: location_name = parsed_query['override_location_name'] - cmd = [WEGO, '--city=%s' % location] + cmd = [WEGO, '--location=%s' % location] if parsed_query.get('inverted_colors'): cmd += ['-inverse'] @@ -94,9 +99,7 @@ def _wego_wrapper(location, parsed_query): if parsed_query.get('use_imperial', False): cmd += ['-imperial'] - if location_name: - cmd += ['-location_name', location_name] - + logging.debug(f'wego cmd: "{cmd}"') proc = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() stdout = stdout.decode("utf-8") @@ -158,12 +161,16 @@ def _htmlize(ansi_output, title, parsed_query): if not parsed_query.get('inverted_colors'): cmd += ["--bg=dark"] + logging.debug(f'Running ansi2html: "{cmd}"') proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate(ansi_output.encode("utf-8")) stdout = stdout.decode("utf-8") stderr = stderr.decode("utf-8") if proc.returncode != 0: + logging.error(f'ansi2html failed: {stderr}') error(stdout + stderr) + else: + logging.debug(f'ansi2html did not fail: "{stderr}" and produced output: {stdout}') if parsed_query.get('inverted_colors'): stdout = stdout.replace( diff --git a/lib/wttr_srv.py b/lib/wttr_srv.py index e7fe484c..8d8bd0bc 100644 --- a/lib/wttr_srv.py +++ b/lib/wttr_srv.py @@ -57,6 +57,7 @@ def show_text_file(name, lang): text = text\ .replace('NUMBER_OF_LANGUAGES', str(len(SUPPORTED_LANGS)))\ .replace('SUPPORTED_LANGUAGES', ' '.join(SUPPORTED_LANGS)) + logging.debug(f'Showing text for {name}') return text def _client_ip_address(request): @@ -199,28 +200,34 @@ def _response(parsed_query, query, fast_mode=False): answer = cache.get(cache_signature) if parsed_query['orig_location'] in PLAIN_TEXT_PAGES: + logging.debug('Returning static data page') answer = show_text_file(parsed_query['orig_location'], parsed_query['lang']) if parsed_query['html_output']: answer = render_template('index.html', body=answer) if answer or fast_mode: + logging.debug('Returning cached result. Or failing on fast_mode.') return answer # at this point, we could not handle the query fast, # so we handle it with all available logic loc = (parsed_query['orig_location'] or "").lower() + logging.debug(f'No luck with cache for "{loc}", getting data for real.') if parsed_query.get("view"): + logging.debug(f'Querying wttr_line: {query}') if not parsed_query.get("location"): parsed_query["location"] = loc - output = wttr_line(query, parsed_query) elif loc == 'moon' or loc.startswith('moon@'): + logging.debug(f'Querying moon') output = get_moon(parsed_query) else: + logging.debug(f'Querying wetter') output = get_wetter(parsed_query) if parsed_query.get('png_filename'): if parsed_query.get("view") != "v3": + logging.debug(f'Rendering results into a png') # originally it was just a usual function call, # but it was a blocking call, so it was moved # to separate threads: @@ -233,6 +240,7 @@ def _response(parsed_query, query, fast_mode=False): if query.get('days', '3') != '0' \ and not query.get('no-follow-line') \ and ((parsed_query.get("view") or "v2")[:2] in ["v2", "v3"]): + logging.debug(f'Follow section not disabled, rendering schwag.') if parsed_query['html_output']: output = add_buttons(output) else: @@ -240,7 +248,7 @@ def _response(parsed_query, query, fast_mode=False): if parsed_query.get('no-terminal', False): message = remove_ansi(message) output += '\n' + message + '\n' - + logging.debug(f'Slow mode complete') return cache.store(cache_signature, output) def parse_request(location, request, query, fast_mode=False): @@ -377,7 +385,9 @@ def _wrap_response(response_text, html_output, json_output, png_filename=None): try: if not response: parsed_query = parse_request(location, request, query) + logging.debug(f'Query: {parsed_query}') response = _response(parsed_query, query) + logging.debug(f'Result: {response}') #if not response or (isinstance(response, str) and not response.strip()): # return RuntimeError("Empty answer") diff --git a/share/docker/supervisord.conf b/share/docker/supervisord.conf index bebbd914..6b01c445 100644 --- a/share/docker/supervisord.conf +++ b/share/docker/supervisord.conf @@ -1,22 +1,16 @@ [supervisord] nodaemon=true -logfile=/var/log/supervisor/supervisord.log pidfile=/var/run/supervisord.pid +redirect_stderr=true [program:srv] command=python3 /app/bin/srv.py -stderr_logfile=/var/log/supervisor/srv-stderr.log -stdout_logfile=/var/log/supervisor/srv-stdout.log +redirect_stderr=true [program:proxy] command=python3 /app/bin/proxy.py -stderr_logfile=/var/log/supervisor/proxy-stderr.log -stdout_logfile=/var/log/supervisor/proxy-stdout.log +redirect_stderr=true [program:geoproxy] command=python3 /app/bin/geo-proxy.py -stderr_logfile=/var/log/supervisor/geoproxy-stderr.log -stdout_logfile=/var/log/supervisor/geoproxy-stdout.log - -[include] -files=/etc/supervisor/conf.d/*.conf +redirect_stderr=true