1.1-RC-1 (#3)

* FIX: removed os calls from python * Support for multiple ralph instances * elk version update (#2) * update documents * fix ralph db encoding * tenant support * fix asset multi tenant * docker multistage * dockerfile cleanup * dockerfile cleanup * dokcer-compose config check * update build system Co-authored-by: plinkert <51674366+plinkert@users.noreply.github.com>
DSecureMe · Jul 20, 2020 · 7e047c8 · 7e047c8
1 parent c2c3509
commit 7e047c8
Show file tree

Hide file tree

Showing 19 changed files with 305 additions and 182 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,28 +1,48 @@
-FROM centos:7.7.1908
-ARG VMC_VERSION=1.0.1-alpha2
+FROM centos:7.7.1908 as builder
+
+ARG VMC_VERSION=1.1-RC-1
 ENV VMC_VERSION=${VMC_VERSION}
 
+
+RUN yum install -y epel-release; \
+    yum install -y python36 python36-devel mariadb-devel gcc; \
+    python3 -m venv /opt/vmc;
+
+ENV PATH="/opt/vmc/bin:$PATH"
+
+RUN pip3.6 install --no-cache-dir vmcenter==${VMC_VERSION}
+
+
+FROM centos:7.7.1908
+
 ENV TZ=Poland
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
+ENV VMC_VERSION=${VMC_VERSION}
+ENV PATH="/opt/vmc/bin:$PATH"
+
 
-LABEL org.label-schema.license="Apache-2.0" \
+LABEL org.label-schema.schema-version="1.1-RC-1" \
+      org.label-schema.license="Apache-2.0" \
       org.label-schema.url="http://dsecure.me"\
       org.label-schema.vendor="DSecure.me" \
       org.label-schema.name="VMC"
 
 COPY root /
+COPY --from=builder /opt/vmc /opt/vmc
 
 RUN yum install -y epel-release; \
-    yum -y update;\
-    yum install -y python36 python36-devel mariadb-devel gcc nginx; \
+    yum -y update; \
+    yum install -y python36 nginx; \
+    mkdir -p /usr/share/vmc/static; \
+    vmc collectstatic --noinput --clear; \
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone; \
-    pip3.6 install vmcenter==${VMC_VERSION}; \
     yum clean all; \
+    rm -rf /var/cache/yum; \
     chmod g=u /etc/passwd; \
-    vmc collectstatic --noinput --clear; \
     chmod +x /usr/bin/entrypoint;
 
+
 EXPOSE 8080
 
 USER 1001

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-v ?= 1.0.1-alpha2
+v ?= 1.1-RC-1
 
 all: build
 

diff --git a/demo/.env b/demo/.env
@@ -1,4 +1,4 @@
-ELK_VERSION=7.3.2
+ELK_VERSION=7.5.0
 POSTGRES_VERSION=11.5
 RABBITMQ_VERSION=3.7.18-alpine
 REDDIS_VERSION=5.0.5

diff --git a/demo/Makefile b/demo/Makefile
@@ -26,6 +26,7 @@ build-elk:
 
 up:
 	sudo sysctl -w vm.max_map_count=262144
+	docker-compose $(VMC) config
 	docker-compose $(VMC) up
 
 demodata:

diff --git a/demo/compose/docker-compose.elastalert.yml b/demo/compose/docker-compose.elastalert.yml
@@ -3,7 +3,7 @@ version: '3.3'
 services:
   elastalert:
     image: bitsensor/elastalert:3.0.0-beta.1
-    restart: always
+    restart: unless-stopped
     depends_on:
       - kibana
       - elasticsearch

diff --git a/demo/compose/docker-compose.ralph.yml b/demo/compose/docker-compose.ralph.yml
@@ -23,7 +23,7 @@ services:
 
   ralph_nginx:
     image: allegro/ralph-static-nginx:20191219.1
-    restart: always
+    restart: unless-stopped
     ports:
       - "8081:80"
     volumes:
@@ -34,6 +34,7 @@ services:
     env_file: ../.env
     volumes:
       - mariadb-data:/var/lib/mysql
+    command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
 
   ralph_inkpy:
     image: allegro/inkpy:0.1.4

diff --git a/demo/config/demo_data.sh b/demo/config/demo_data.sh
@@ -6,18 +6,19 @@ docker exec -it compose_web_1 ralphctl demodata
 docker exec -it compose_web_1 python3 /test_data/generate_ips.py
 
 echo "Elastalert: Create Indexes"
-docker exec -it compose_elastalert_1 /usr/bin/elastalert-create-index --config /opt/elastalert/config.yaml
+#docker exec -it compose_elastalert_1 /usr/bin/elastalert-create-index --config /opt/elastalert/config.yaml
 
 echo "HIVE: create database (superuser login: admin, password: admin)"
-docker exec -it compose_hive_elasticsearch_1 chmod +x /test_data/load.sh
-docker exec -it compose_hive_elasticsearch_1 /test_data/load.sh
+#docker exec -it compose_hive_elasticsearch_1 chmod +x /test_data/load.sh
+#docker exec -it compose_hive_elasticsearch_1 /test_data/load.sh
 
 echo "VMC: Load data (superuser login: admin, password: adminadmin)"
 docker exec -it compose_admin_1 vmc loaddata /test_data/demo_data.json
+docker exec -it compose_admin_1 vmc create_index
 
 echo "Kibana: Import Sample Dashboards and KPIs"
-docker exec -it compose_kibana_1 chmod +x /test_data/load.sh
-docker exec -it compose_kibana_1 /test_data/load.sh
+#docker exec -it compose_kibana_1 chmod +x /test_data/load.sh
+#docker exec -it compose_kibana_1 /test_data/load.sh
 
-echo "VMC: Import Assets from Ralph"
+echo "VMC: Prepare demo data"
 docker exec -it compose_admin_1 python3 -W ignore /test_data/generate_vulns.py
diff --git a/...Host with high vulnerability scorev2.yaml → ...Host with high vulnerability scorev2.yaml b/...Host with high vulnerability scorev2.yaml → ...Host with high vulnerability scorev2.yaml
diff --git a/...set whith empty field business owner.yaml → ...New asset whith empty business owner.yaml b/...set whith empty field business owner.yaml → ...New asset whith empty business owner.yaml
@@ -8,8 +8,8 @@ realert:
     minutes: 0
 
 filter:
-- term:
-    business_owner: "UNKNOWN"
+  - query:
+      query_string: {query: 'not __exists__:business_owner.name'}
 
 hive_connection:
   hive_host: http://hive

diff --git a/...vulnerability with available exploit.yaml → ...vulnerability with available exploit.yaml b/...vulnerability with available exploit.yaml → ...vulnerability with available exploit.yaml
diff --git a/demo/config/kibana/kibana.yml b/demo/config/kibana/kibana.yml
@@ -8,3 +8,4 @@ xpack.ml.enabled: false
 xpack.logstash.enabled: false
 xpack.infra.enabled: false
 xpack.uptime.enabled: false
+xpack.apm.ui.enabled: false
diff --git a/demo/config/ralph/generate_ips.py b/demo/config/ralph/generate_ips.py
@@ -38,11 +38,23 @@
 USED_MAC = []
 
 IMPACT = ['LOW', 'MEDIUM', 'HIGH', 'NOT_DEFINED']
-OS_LIST = ['Windows 2000', 'Windows XP',  'Windows Vista', 'Windows 7', 'Windows 8', 'Windows 10',
-           'Windows Server', 'Windows Home Server', 'Ubuntu 14.10', 'Ubuntu 18.10', 'Redhat 7', 'Centos 8',
-           'Debian', 'Kali']
+IMPACT_WEIGHTS = [0.25, 0.25, 0.25, 0.25]
+
+OS_FAMILY = {
+    'Redhat': ['Redhat 5', 'Redhat 6', 'Redhat 7'],
+    'Debian': ['Debian 8', 'Debian 9', 'Debian 10'],
+    'Windows': ['Windows Server 2012', 'Windows Server 2016', 'Windows Server 2019'],
+    'IBM': ['IBM AIX 5', 'IBM AIX 6', 'IBM AIX 7']
+}
+
+"""
+Probability distribution of the images used modeled on:
+https://ieeexplore.ieee.org/abstract/document/6195556
+"""
+OS_FAMILY_WEIGHTS = [0.53, 0.07, 0.25, 0.15]
 
 USER_MODEL = get_user_model()
+MIN_IP_COUNT = 1000
 
 
 def _generate_mac():
@@ -59,9 +71,9 @@ def _generate_mac():
 
 def _random_ip_address():
     while True:
-        octets = []
-        for x in range(4):
-            octets.append(str(random.randint(0, 255)))
+        octets = ['192', '168']
+        for x in range(2):
+            octets.append(str(random.randint(1, 254)))
         ip = '.'.join(octets)
         if ip not in USED_IPS:
             USED_IPS.append(ip)
@@ -77,51 +89,48 @@ def main():
     )
     confidentiality_field = CustomField.objects.create(
         name='confidentiality',
-        type=CustomFieldTypes.STRING,
+        type=CustomFieldTypes.CHOICE,
         choices='LOW|MEDIUM|HIGH|NOT_DEFINED',
     )
     integrity_field = CustomField.objects.create(
         name='integrity',
-        type=CustomFieldTypes.STRING,
+        type=CustomFieldTypes.CHOICE,
         choices='LOW|MEDIUM|HIGH|NOT_DEFINED',
     )
     availability_field = CustomField.objects.create(
         name='availability',
-        type=CustomFieldTypes.STRING,
+        type=CustomFieldTypes.CHOICE,
         choices='LOW|MEDIUM|HIGH|NOT_DEFINED',
     )
     content_type = ContentType.objects.get_for_model(DataCenterAsset)
-    print('Generating random data for addesses', dc_assets.count())
 
-    for asset in dc_assets:
-        eth = Ethernet.objects.create(base_object=asset.asset, mac=_generate_mac())
-        IPAddress.objects.create(address=_random_ip_address(), ethernet=eth, hostname=asset.hostname)
-        if random.choice([True, False]):
-            asset.custom_fields.add(CustomFieldValue.objects.create(
-                custom_field=confidentiality_field,
-                value=random.choice(IMPACT),
-                object_id=asset.pk,
-                content_type=content_type
-            ))
-            asset.custom_fields.add(CustomFieldValue.objects.create(
-                custom_field=integrity_field,
-                value=random.choice(IMPACT),
-                object_id=asset.pk,
-                content_type=content_type
-            ))
-            asset.custom_fields.add(CustomFieldValue.objects.create(
-                custom_field=availability_field,
-                value=random.choice(IMPACT),
-                object_id=asset.pk,
-                content_type=content_type
-            ))
-        if random.choice([True, False]):
-            asset.custom_fields.add(CustomFieldValue.objects.create(
-                custom_field=os_field,
-                value=random.choice(random.choice(OS_LIST)),
-                object_id=asset.pk,
-                content_type=content_type
-            ))
+    print('Generating random data for addesses', dc_assets.count())
+    for idx, asset in enumerate(dc_assets):
+        asset.custom_fields.add(CustomFieldValue.objects.create(
+            custom_field=confidentiality_field,
+            value=random.choices(IMPACT, IMPACT_WEIGHTS)[0],
+            object_id=asset.pk,
+            content_type=content_type
+        ))
+        asset.custom_fields.add(CustomFieldValue.objects.create(
+            custom_field=integrity_field,
+            value=random.choices(IMPACT, IMPACT_WEIGHTS)[0],
+            object_id=asset.pk,
+            content_type=content_type
+        ))
+        asset.custom_fields.add(CustomFieldValue.objects.create(
+            custom_field=availability_field,
+            value=random.choices(IMPACT, IMPACT_WEIGHTS)[0],
+            object_id=asset.pk,
+            content_type=content_type
+        ))
+        os_family = random.choices(list(OS_FAMILY.keys()), OS_FAMILY_WEIGHTS)[0]
+        asset.custom_fields.add(CustomFieldValue.objects.create(
+            custom_field=os_field,
+            value=random.choices(OS_FAMILY[os_family])[0],
+            object_id=asset.pk,
+            content_type=content_type
+        ))
 
         if random.random() > 0.90 and not asset.service_env.service.business_owners.exists():
             bo = USER_MODEL.objects.order_by('?').first()
@@ -133,6 +142,14 @@ def main():
 
         asset.save()
 
+    while len(USED_IPS) <= MIN_IP_COUNT:
+        for idx, asset in enumerate(dc_assets):
+            eth = Ethernet.objects.create(base_object=asset.asset, mac=_generate_mac())
+            IPAddress.objects.create(address=_random_ip_address(), ethernet=eth, hostname=asset.hostname)
+        if len(USED_IPS) // 100:
+            print(len(USED_IPS))
+
+    print(F'Generated ips: {len(USED_IPS)}')
     print('Generation done')
 
 

diff --git a/demo/config/vmc/demo_data.json b/demo/config/vmc/demo_data.json
@@ -17,93 +17,15 @@
       "user_permissions": []
     }
   },
-  {
-    "model": "django_celery_beat.crontabschedule",
-    "pk": 1,
-    "fields": {
-      "minute": "0",
-      "hour": "6",
-      "day_of_week": "*",
-      "day_of_month": "*",
-      "month_of_year": "*",
-      "timezone": "UTC"
-    }
-  },
-  {
-    "model": "django_celery_beat.crontabschedule",
-    "pk": 2,
-    "fields": {
-      "minute": "*",
-      "hour": "*",
-      "day_of_week": "*",
-      "day_of_month": "*",
-      "month_of_year": "*",
-      "timezone": "UTC"
-    }
-  },
-  {
-    "model": "django_celery_beat.periodictasks",
-    "pk": 1,
-    "fields": {
-      "last_update": "2019-04-20T21:18:38.990Z"
-    }
-  },
-  {
-    "model": "django_celery_beat.periodictask",
-    "pk": 1,
-    "fields": {
-      "name": "Update CVE and CWE every night",
-      "task": "vmc.knowledge_base.tasks.update_cve_cwe",
-      "interval": null,
-      "crontab": 1,
-      "solar": null,
-      "args": "[]",
-      "kwargs": "{}",
-      "queue": null,
-      "exchange": null,
-      "routing_key": null,
-      "priority": null,
-      "expires": null,
-      "one_off": false,
-      "start_time": null,
-      "enabled": true,
-      "last_run_at": null,
-      "total_run_count": 0,
-      "date_changed": "2019-04-20T21:17:55.202Z",
-      "description": ""
-    }
-  },
-    {
-    "model": "django_celery_beat.periodictask",
-    "pk": 2,
-    "fields": {
-      "name": "Update Assets",
-      "task": "vmc.ralph.tasks.load_all_assets",
-      "interval": null,
-      "crontab": 2,
-      "solar": null,
-      "args": "[]",
-      "kwargs": "{}",
-      "queue": null,
-      "exchange": null,
-      "routing_key": null,
-      "priority": null,
-      "expires": null,
-      "one_off": false,
-      "start_time": null,
-      "enabled": true,
-      "last_run_at": null,
-      "total_run_count": 0,
-      "date_changed": "2019-04-20T21:17:55.202Z",
-      "description": ""
-    }
-  },
   {
     "model": "ralph.config",
     "pk": 1,
     "fields": {
       "name": "Demo Ralph",
-      "url": "http://web:8000",
+      "schema": "http",
+      "host": "web",
+      "port": 8000,
+      "insecure": true,
       "username": "ralph",
       "password": "ralph",
       "created_date": "2019-03-06T22:37:31.540Z",