#46 #47 #66 Connectors implementation and many global enhancements

.circleci/config.yml

@@ -7,6 +7,11 @@ jobs:
       - image: circleci/node:11-stretch
     steps:
       - checkout
+      - run:
+          name: "Pull Submodules"
+          command: |
+            git submodule init
+            git submodule update --remote
       - run: 
           working_directory: ~/opencti/opencti-front
           command: yarn install
@@ -66,6 +71,7 @@ jobs:
       - run: git remote add opencti ssh://[email protected]:/~/git-OLOszi5F.git && git push opencti master
       - run: mkdir release
       - run: cp -a opencti-graphql release/opencti
+      - run: cp -a opencti-integration release/opencti/integration
       - run: cp -a opencti-worker release/opencti/worker
       - run: 
           working_directory: ~/opencti/release

README.md

@@ -1,16 +1,17 @@
 ![](docs/logo.png)
 ---
 [![Website](https://img.shields.io/badge/website-opencti.io-blue.svg)](https://www.opencti.io)
-[![CircleCI](https://circleci.com/gh/LuatixHQ/opencti.svg?style=shield&circle-token=71bf657a43ec48f8433ece2f4879eadaf920d79a)](https://circleci.com/gh/LuatixHQ/opencti)
-[![GitHub release](https://img.shields.io/github/release/LuatixHQ/opencti.svg)](https://github.com/LuatixHQ/opencti/releases/latest)
+[![CircleCI](https://circleci.com/gh/OpenCTI-Platform/opencti.svg?style=shield&circle-token=71bf657a43ec48f8433ece2f4879eadaf920d79a)](https://circleci.com/gh/OpenCTI-Platform/opencti)
+[![GitHub release](https://img.shields.io/github/release/OpenCTI-Platform/opencti.svg)](https://github.com/OpenCTI-Platform/opencti/releases/latest)
+[![Slack Status](https://slack.luatix.org/badge.svg)](https://slack.luatix.org)
 
 OpenCTI is an open source platform allowing organizations to manage their cyber threat intelligence knowledge, investigations and indicators of compromise. OpenCTI can be integrated with other applications such as [Maltego](https://www.paterva.com/web7/buy/maltego-clients/maltego-ce.php), [MISP](https://www.misp-project.org/), [CORTEX](https://github.com/TheHive-Project/Cortex) and many other STIX2 compliant products. It has been designed as a modern web application including a GraphQL API and an UX oriented frontend.
 
 ![Screenshot](docs/screenshot.png "OpenCTI")
 
 ## Releases download
 
-The releases are available on the [Github releases page](https://github.com/LuatixHQ/opencti/releases). You can also access to the [rolling release package](https://releases.opencti.io) generated from the mater branch of the repository.
+The releases are available on the [Github releases page](https://github.com/OpenCTI-Platform/opencti/releases). You can also access to the [rolling release package](https://releases.opencti.io) generated from the mater branch of the repository.
 
 ## Installation
 
@@ -34,12 +35,12 @@ The [Grakn](https://github.com/graknlabs/grakn) knowledge graph database provide
 
 ### Status & bugs
 
-Currently OpenCTI is under heavy development, if you wish to report bugs or ask for new features, you can directly use the [Github issues module](https://github.com/LuatixHQ/opencti/issues).
+Currently OpenCTI is under heavy development, if you wish to report bugs or ask for new features, you can directly use the [Github issues module](https://github.com/OpenCTI-Platform/opencti/issues).
 
 ### Discussion
 
 If you need support or you wish to engage a discussion about the OpenCTI platform, feel free to join us on our [Slack channel](https://slack.luatix.org). You can also send us an email to [email protected].
 
-### About
+## About
 
 OpenCTI is a product powered by the collaboration of the [French national cybersecurity agency (ANSSI)](https://ssi.gouv.fr), the [CERT-EU](https://cert.europa.eu) and the [Luatix](https://www.luatix.org) non-profit organization.

docker/docker-compose.yml

@@ -21,6 +21,15 @@ services:
     volumes:
       - "./config:/opt/opencti/shared_config:rw"
     restart: always
+  integration:
+    image: luatix/opencti-integration
+    build:
+     context: ./integration
+    environment:
+      - RUN_USER=root
+    volumes:
+      - "./config:/opt/opencti/shared_config:rw"
+    restart: always    
   opencti:
     image: luatix/opencti
     build:

docker/integration/Dockerfile

@@ -0,0 +1,26 @@
+# Use the official Docker Hub Ubuntu 18.04 base image
+FROM ubuntu:18.04
+
+# Update the base image
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -y upgrade && apt-get -y dist-upgrade
+
+# Setup install environment and OpenCTI dependencies
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip curl sudo rsyslog wget netcat locales
+
+# Download archive
+RUN wget -O /opt/opencti.tar.gz https://releases.opencti.io/opencti-20190623.tar.gz
+RUN cd /opt && tar xvfz opencti.tar.gz
+
+# Set the locale
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8  
+ENV LANGUAGE en_US:en  
+ENV LC_ALL en_US.UTF-8 
+
+# Prepare configuration
+RUN cd /opt/opencti/integration && ln -s /opt/opencti/shared_config/config.yml .
+
+# Expose and entrypoint
+COPY entrypoint.sh /
+RUN chmod +x /entrypoint.sh 
+ENTRYPOINT ["/entrypoint.sh"]

docker/integration/entrypoint.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# Start log
+/etc/init.d/rsyslog start
+
+# Install Python modules
+cd /opt/opencti/integration
+sudo -H -u ${RUN_USER} pip3 install -r requirements.txt
+
+# Install Python modules of each connectors
+sudo -H -u ${RUN_USER} find connectors -name requirements.txt -exec pip3 install -r {} \;
+
+# Check configuration
+while [ ! -f /opt/opencti/shared_config/token ]
+do
+  echo "Waiting for token in shared config..."
+  sleep 2
+done
+
+# Replace the token in the configuration
+sed -i -e "s/REPLACE_API_KEY/$(cat /opt/opencti/shared_config/token)/g" /opt/opencti/integration/config.yml.sample
+cp /opt/opencti/integration/config.yml.sample /opt/opencti/shared_config/config.yml
+
+# Chown the application
+chown -R ${RUN_USER} /opt/opencti
+
+# Start
+cd /opt/opencti/integration
+sudo -H -u ${RUN_USER} python3 connectors_scheduler.py

docker/opencti/Dockerfile

@@ -10,7 +10,7 @@ RUN curl -sL https://deb.nodesource.com/setup_11.x | sudo -E bash -
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs
 
 # Download archive
-RUN wget -O /opt/opencti.tar.gz https://releases.opencti.io/opencti-20190619.tar.gz
+RUN wget -O /opt/opencti.tar.gz https://releases.opencti.io/opencti-20190623.tar.gz
 RUN cd /opt && tar xvfz opencti.tar.gz
 
 # Expose and entrypoint

docker/worker/Dockerfile

@@ -8,7 +8,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -y upgrade && apt-g
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip curl sudo rsyslog wget netcat locales
 
 # Download archive
-RUN wget -O /opt/opencti.tar.gz https://releases.opencti.io/opencti-20190619.tar.gz
+RUN wget -O /opt/opencti.tar.gz https://releases.opencti.io/opencti-20190623.tar.gz
 RUN cd /opt && tar xvfz opencti.tar.gz
 
 # Set the locale

docs/Development.md

@@ -22,7 +22,7 @@ $ sudo apt-get update && sudo apt-get install yarn
 *Download the application files*:
 ```bash
 $ mkdir /path/to/your/app && cd /path/to/your/app
-$ git clone https://github.com/Luatix/opencti.git
+$ git clone --recursive https://github.com/Luatix/opencti.git
 $ cd opencti
 ```
 

opencti-front/src/components/ItemIcon.js

@@ -78,6 +78,9 @@ const iconSelector = (type, variant, fontSize, color) => {
     case 'email-address':
     case 'mutex':
     case 'file':
+    case 'file-md5':
+    case 'file-sha1':
+    case 'file-sha256':
       return <Tag style={style} fontSize={fontSize} />;
     default:
       return <Help style={style} fontSize={fontSize} />;

opencti-front/src/private/Root.js

@@ -10,7 +10,7 @@ import { ConnectedDocumentTitle } from '../components/AppDocumentTitle';
 import TopBar from './components/nav/TopBar';
 import LeftBar from './components/nav/LeftBar';
 import Dashboard from './components/Dashboard';
-import StixDomainEntities from './components/StixDomainEntities';
+import Search from './components/Search';
 import Workspaces from './components/Workspaces';
 import StixObservables from './components/StixObservables';
 import RootStixObservable from './components/stix_observable/Root';
@@ -44,6 +44,7 @@ import Cities from './components/Cities';
 import Organizations from './components/Organizations';
 import RootOrganization from './components/organization/Root';
 import Persons from './components/Persons';
+import Connectors from './components/Connectors';
 import Settings from './components/Settings';
 import Users from './components/Users';
 import Groups from './components/Groups';
@@ -118,7 +119,7 @@ class Root extends Component {
                           exact
                           path="/dashboard/search/:keyword"
                           render={routeProps => (
-                            <StixDomainEntities {...routeProps} me={props.me} />
+                            <Search {...routeProps} me={props.me} />
                           )}
                         />
                         <BoundaryRoute
@@ -361,6 +362,13 @@ class Root extends Component {
                             <RootWorkspace {...routeProps} workspaceType="investigate" />
                           )}
                         />
+                        <BoundaryRoute
+                          exact
+                          path="/dashboard/connectors"
+                          render={routeProps => (
+                            <Connectors {...routeProps} type="importer" />
+                          )}
+                        />
                         <BoundaryRoute
                           exact
                           path="/dashboard/settings"