Implement TeslaFi import

Closes #316
teslamate-org · Feb 7, 2020 · f61315c · f61315c
1 parent 7322e9d
commit f61315c
Show file tree

Hide file tree

Showing 52 changed files with 4,589 additions and 185 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -8,3 +8,5 @@ README*
 test/
 priv/static/
 **/node_modules
+import/
+teslamate.bck
diff --git a/.gitignore b/.gitignore
@@ -43,6 +43,7 @@ npm-debug.log
 config/*.env
 
 /deploy/
+/import/
 conf/wizzy.json
 
 .env
@@ -54,4 +55,4 @@ conf/wizzy.json
 
 # other stuff created by my build env?!
 .elixir_ls/
-assets/package-lock.json
+assets/package-lock.json
diff --git a/README.md b/README.md
@@ -41,6 +41,7 @@ A powerful, self-hosted data logger for your Tesla.
 - Geo-fencing feature to create custom locations
 - Supports multiple vehicles per Tesla Account
 - Charge cost tracking
+- Import from TeslaFi
 
 ## Screenshots
 
@@ -68,6 +69,8 @@ The full TeslaMate documentation is available on [Read the Docs](https://teslama
   - [Shortcuts Setup (iOS)](https://teslamate.readthedocs.io/en/latest/configuration/guides/shortcuts.html)
   - [Tasker Setup (Android)](https://teslamate.readthedocs.io/en/latest/configuration/guides/tasker.html)
   - [MacroDroid Setup (Android)](https://teslamate.readthedocs.io/en/latest/configuration/guides/macro_droid.html)
+- Import
+  - [TeslaFi](https://teslamate.readthedocs.io/en/latest/import/teslafi.html)
 - Integrations
   - [HomeAssistant](https://teslamate.readthedocs.io/en/latest/integrations/home_assistant.html)
   - [MQTT](https://teslamate.readthedocs.io/en/latest/integrations/mqtt.html)

diff --git a/assets/css/app.scss b/assets/css/app.scss
@@ -14,9 +14,9 @@ $fullhd-enabled: false;
 @import "~bulma/sass/base/_all";
 @import "~bulma/sass/components/breadcrumb";
 @import "~bulma/sass/components/card";
+@import "~bulma/sass/components/dropdown";
 @import "~bulma/sass/components/message";
 @import "~bulma/sass/components/navbar";
-@import "~bulma/sass/components/dropdown";
 @import "~bulma/sass/elements/box";
 @import "~bulma/sass/elements/button";
 @import "~bulma/sass/elements/container";
@@ -253,3 +253,17 @@ $positions: ('top','left','bottom','right');
 .field.is-horizontal .field-label {
   min-width: 200px;
 }
+
+main.full-height {
+  min-height: calc(100vh - 102px);
+  min-height: calc((var(--vh, 1vh) * 100) - 102px);
+}
+
+.footer {
+  height: 50px;
+  opacity: 0.8;
+
+  .button.is-text {
+    text-decoration: initial;
+  }
+}
diff --git a/assets/js/main.js b/assets/js/main.js
@@ -6,3 +6,12 @@ document.querySelector(".navbar-burger").addEventListener("click", function() {
 
 // Fix sticky hover on iOS
 document.addEventListener("click", () => 0);
+
+// Address dynamic viewport units on mobile
+function setCustomVh() {
+  let vh = window.innerHeight * 0.01;
+  document.documentElement.style.setProperty("--vh", `${vh}px`);
+}
+
+window.addEventListener("resize", setCustomVh);
+setCustomVh();
diff --git a/config/releases.exs b/config/releases.exs
@@ -19,6 +19,25 @@ defmodule Util do
   def parse_check_origin!("false"), do: false
   def parse_check_origin!(hosts) when is_binary(hosts), do: String.split(hosts, ",")
   def parse_check_origin!(hosts), do: raise("Invalid check_origin option: #{inspect(hosts)}")
+
+  def validate_import_dir(nil), do: nil
+
+  def validate_import_dir(path) do
+    case File.ls(path) do
+      {:ok, [_ | _]} ->
+        path
+
+      {:ok, []} ->
+        nil
+
+      {:error, :enoent} ->
+        nil
+
+      {:error, reason} ->
+        IO.puts("Cannot access directory '#{path}': #{inspect(reason)}")
+        nil
+    end
+  end
 end
 
 config :teslamate, TeslaMate.Repo,
@@ -27,7 +46,8 @@ config :teslamate, TeslaMate.Repo,
   database: System.fetch_env!("DATABASE_NAME"),
   hostname: System.fetch_env!("DATABASE_HOST"),
   port: System.get_env("DATABASE_PORT", "5432"),
-  pool_size: System.get_env("DATABASE_POOL_SIZE", "8") |> String.to_integer()
+  pool_size: System.get_env("DATABASE_POOL_SIZE", "10") |> String.to_integer(),
+  timeout: System.get_env("DATABASE_TIMEOUT", "60000") |> String.to_integer()
 
 config :teslamate, TeslaMateWeb.Endpoint,
   http: [:inet6, port: System.get_env("PORT", "4000")],
@@ -55,3 +75,6 @@ config :logger, :console,
   metadata: [:car_id]
 
 config :teslamate, :srtm_cache, System.get_env("SRTM_CACHE", ".srtm_cache")
+
+config :teslamate,
+  import_directory: System.get_env("IMPORT_DIR", "import") |> Util.validate_import_dir()
diff --git a/docs/configuration/environment_variables.md b/docs/configuration/environment_variables.md
@@ -9,15 +9,17 @@ TeslaMate accepts the following environment variables for runtime configuration:
 | **DATABASE_NAME**                 | The database to connect to (**required**)                                                                                                                                                                                                                                                                                                    |               |
 | **DATABASE_HOST**                 | Hostname of the database server (**required**)                                                                                                                                                                                                                                                                                               |               |
 | **DATABASE_PORT**                 | Port of the database server                                                                                                                                                                                                                                                                                                                  | 5432          |
-| **DATABASE_POOL_SIZE**            | Size of the database connection pool                                                                                                                                                                                                                                                                                                         | 8             |
+| **DATABASE_POOL_SIZE**            | Size of the database connection pool                                                                                                                                                                                                                                                                                                         | 10            |
+| **DATABASE_TIMEOUT**              | The time in milliseconds to wait for database query calls to finish                                                                                                                                                                                                                                                                          | 60000         |
 | **VIRTUAL_HOST**                  | Host part used for generating URLs throughout the app                                                                                                                                                                                                                                                                                        | localhost     |
 | **CHECK_ORIGIN**                  | Configures whether to check the origin header or not. May be `true` (**recommended**), `false` (_default_) or a comma-separated list of hosts that are allowed (e.g. `https://example.com,//another.com:8080`). Hosts also support wildcards. It defaults to true and, in such case, it will check against the host value in `VIRTUAL_HOST`. | false         |
 | **PORT**                          | Port where the web interface is exposed                                                                                                                                                                                                                                                                                                      | 4000          |
 | **DISABLE_MQTT**                  | Disables the MQTT feature if `true`                                                                                                                                                                                                                                                                                                          | false         |
 | **MQTT_HOST**                     | Hostname of the broker (**required** unless DISABLE_MQTT is `true`)                                                                                                                                                                                                                                                                          |               |
-| **MQTT_USERNAME**                 | Username _(optional)_                                                                                                                                                                                                                                                                                                                        |               |
-| **MQTT_PASSWORD**                 | Password _(optional)_                                                                                                                                                                                                                                                                                                                        |               |
-| **MQTT_TLS**                      | Enables TLS if `true` _(optional)_                                                                                                                                                                                                                                                                                                           | false         |
-| **MQTT_TLS_ACCEPT_INVALID_CERTS** | Accepts invalid certificates if `true` _(optional)_                                                                                                                                                                                                                                                                                          | false         |
-| **MQTT_NAMESPACE**                | Inserts a custom namespace into the MQTT topic _(optional)_. For example, with `MQTT_NAMESPACE=account_0`: `teslamate/account_0/cars/$car_id/state`.                                                                                                                                                                                         |               |
+| **MQTT_USERNAME**                 | Username                                                                                                                                                                                                                                                                                                                                     |               |
+| **MQTT_PASSWORD**                 | Password                                                                                                                                                                                                                                                                                                                                     |               |
+| **MQTT_TLS**                      | Enables TLS if `true`                                                                                                                                                                                                                                                                                                                        | false         |
+| **MQTT_TLS_ACCEPT_INVALID_CERTS** | Accepts invalid certificates if `true`                                                                                                                                                                                                                                                                                                       | false         |
+| **MQTT_NAMESPACE**                | Inserts a custom namespace into the MQTT topic . For example, with `MQTT_NAMESPACE=account_0`: `teslamate/account_0/cars/$car_id/state`.                                                                                                                                                                                                     |               |
+| **IMPORT_DIR**                    | The path of the directory for the import of data (e.g. TeslaFi)                                                                                                                                                                                                                                                                              | ./import      |
 | **TZ**                            | Used to establish the local time zone, e.g. to use the local time in logs. See [List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones).                                                                                                                                                               |               |
diff --git a/docs/import/teslafi.md b/docs/import/teslafi.md
@@ -0,0 +1,28 @@
+# Import from TeslaFi (BETA)
+
+## Requirements
+
+- **CREATE A [BACKUP](../maintenance/backup_restore.html) OF YOUR DATA!**
+
+- If you have been using TeslaMate prior to the release of version 1.16, the [docker-compose.yml](../installation/docker.html) needs to be updated. Add the following volume mapping to the `teslamate` service:
+
+  ```YAML
+  services:
+    teslamate:
+      # ...
+      volumes:
+        - ./import:/opt/app/import
+  ```
+
+- Export your TeslaFi data as CSV by month: `Settings -> Account -> "Download TeslaFi Data"`
+
+## Instructions
+
+1. Copy the exported CSV files into a **directory named `import`\*** next to the _docker-compose.yml_
+2. **Restart** the teslamate service and open the TeslaMate admin interface. Now the import form should be displayed instead of the vehicle summary.
+3. Since the raw data is in the local timezone (assigned by the home address in the TeslaFi settings page) you need to **select your local timezone**. Then start the import.
+4. On low-end hardware like the Raspberry Pi the import may take multiple hours, depending on the amount of data. If there is an overlap between the already existing TeslaMate and TeslaFi data, only the data prior to the first TeslaMate data will be imported. After the import is complete, remove or **empty the `import` directory** and **restart** the `teslamate` service.
+
+Since the exported CSV files do not contain addresses, they are added automatically afterwards. So please note that not all addresses are visible immediately after the import/restarting. Depending on the amount of data imported, it may take a while before they appear. The same applies to elevation data.
+
+_\* The path of the import directory can be customized via the [`IMPORT_DIR` environment variable](../configuration/environment_variables.html)._
diff --git a/docs/index.rst b/docs/index.rst
@@ -23,6 +23,12 @@ A powerful, self-hosted data logger for your Tesla.
    configuration/guides/tasker
    configuration/guides/macro_droid
 
+.. toctree::
+   :maxdepth: 1
+   :caption: Import
+
+   import/teslafi.md
+
 .. toctree::
    :maxdepth: 1
    :caption: Integrations

diff --git a/docs/installation/docker.md b/docs/installation/docker.md
@@ -31,6 +31,8 @@ services:
       - MQTT_HOST=mosquitto
     ports:
       - 4000:4000
+    volumes:
+      - ./import:/opt/app/import
     cap_drop:
       - all
 

diff --git a/docs/installation/docker_advanced.md b/docs/installation/docker_advanced.md
@@ -36,6 +36,8 @@ services:
       - VIRTUAL_HOST=${FQDN_TM}
       - CHECK_ORIGIN=true
       - TZ={$TM_TZ}
+    volumes:
+      - ./import:/opt/app/import
     labels:
       - 'traefik.enable=true'
       - 'traefik.port=4000'

diff --git a/docs/installation/docker_advanced_apache.md b/docs/installation/docker_advanced_apache.md
@@ -37,6 +37,8 @@ services:
       - VIRTUAL_HOST=${FQDN_TM}
       - CHECK_ORIGIN=true
       - TZ={$TM_TZ}
+    volumes:
+      - ./import:/opt/app/import
     ports:
       - 127.0.0.1:4000:4000
     cap_drop:

diff --git a/lib/tesla_api/vehicle.ex b/lib/tesla_api/vehicle.ex
@@ -23,18 +23,18 @@ defmodule TeslaApi.Vehicle do
             vehicle_state: nil
 
   def list(%Auth{token: token}) do
-    TeslaApi.get("/api/1/vehicles", token, transform: &vehicle/1)
+    TeslaApi.get("/api/1/vehicles", token, transform: &result/1)
   end
 
   def get(%Auth{token: token}, id) do
-    TeslaApi.get("/api/1/vehicles/#{id}", token, transform: &vehicle/1)
+    TeslaApi.get("/api/1/vehicles/#{id}", token, transform: &result/1)
   end
 
   def get_with_state(%Auth{token: token}, id) do
-    TeslaApi.get("/api/1/vehicles/#{id}/vehicle_data", token, transform: &vehicle/1)
+    TeslaApi.get("/api/1/vehicles/#{id}/vehicle_data", token, transform: &result/1)
   end
 
-  defp vehicle(v) do
+  def result(v) do
     %__MODULE__{
       id: v["id"],
       vehicle_id: v["vehicle_id"],

diff --git a/lib/teslamate/application.ex b/lib/teslamate/application.ex
@@ -10,18 +10,37 @@ defmodule TeslaMate.Application do
     :ok = :telemetry.detach({Phoenix.Logger, [:phoenix, :socket_connected]})
     :ok = :telemetry.detach({Phoenix.Logger, [:phoenix, :channel_joined]})
 
-    [
-      TeslaMate.Repo,
-      TeslaMate.Api,
-      TeslaMate.Locations,
-      TeslaMateWeb.Endpoint,
-      TeslaMate.Terrain,
-      TeslaMate.Vehicles,
-      if(mqtt_enabled?(), do: TeslaMate.Mqtt),
-      TeslaMate.Repair
-    ]
-    |> Enum.reject(&is_nil/1)
-    |> Supervisor.start_link(strategy: :one_for_one, name: TeslaMate.Supervisor)
+    Supervisor.start_link(children(), strategy: :one_for_one, name: TeslaMate.Supervisor)
+  end
+
+  defp children do
+    mqtt_enabled? = !is_nil(Application.get_env(:teslamate, :mqtt))
+
+    case Application.get_env(:teslamate, :import_directory) do
+      nil ->
+        [
+          TeslaMate.Repo,
+          TeslaMate.Api,
+          TeslaMate.Locations,
+          TeslaMateWeb.Endpoint,
+          TeslaMate.Terrain,
+          TeslaMate.Vehicles,
+          if(mqtt_enabled?, do: TeslaMate.Mqtt),
+          TeslaMate.Repair
+        ]
+        |> Enum.reject(&is_nil/1)
+
+      import_directory ->
+        [
+          TeslaMate.Repo,
+          TeslaMate.Api,
+          TeslaMate.Locations,
+          TeslaMateWeb.Endpoint,
+          {TeslaMate.Terrain, disabled: true},
+          {TeslaMate.Repair, limit: 250},
+          {TeslaMate.Import, directory: import_directory}
+        ]
+    end
   end
 
   # Tell Phoenix to update the endpoint configuration
@@ -30,6 +49,4 @@ defmodule TeslaMate.Application do
     TeslaMateWeb.Endpoint.config_change(changed, removed)
     :ok
   end
-
-  defp mqtt_enabled?, do: !is_nil(Application.get_env(:teslamate, :mqtt))
 end