Merge branch 'merger-hotfix-instant-value-copy' of https://github.com…

…/torodb/stampede into torodb-476
torodb · Jun 27, 2017 · 5815581 · 5815581
2 parents d55fbdb + 7891055
commit 5815581
Show file tree

Hide file tree

Showing 6 changed files with 179 additions and 15 deletions.
diff --git a/documentation/docs/configuration/mongodb-connectivity.md b/documentation/docs/configuration/mongodb-connectivity.md
@@ -10,12 +10,11 @@ replication:
   syncSource: localhost:27017
   auth:
     mode: negotiate
-    user: mymongouser
-    source: mymongosource
+    user: stampede
+    source: admin
   ssl:
     enabled: true
-    allowInvalidHostnames: false
-    caFile: mycafile.pem
+    caFile: rootCA.pem
 ```
 
 ## Replicate from a MongoDB Sharded Cluster
@@ -25,6 +24,7 @@ In the replication section of the yml config file add a shards item with the lis
 
 ```json
 replication:
+  replSetName: shard
   shards:
     - replSetName: shard1
       syncSource: localhost:27020
@@ -33,3 +33,133 @@ replication:
     - replSetName: shard3
       syncSource: localhost:27040
 ```
+
+If `/replication/shards/<index>/replSetName` is not specified, `/replication/replSetName` will be used. This mechanism of property value merging is valid for properties in the following sections:
+
+* `/replication/shards/<index>/ssl` will default to `/replication/ssl`
+* `/replication/shards/<index>/auth` will default to `/replication/auth`
+
+## Connect using Secure Socket Layer
+
+To enable SSL connectivity to MongoDB you have to make sure [MongoDB is correctly configured](https://docs.mongodb.com/manual/tutorial/configure-ssl/). 
+If the MongoDB certificate is not issued by a known Certification Authority you have to copy the CA file in a path accessible by ToroDB Stampede. 
+For this example we assume the CA file is `rootCA.pem`. For testing purpose you may want to set property `/replication/ssl/allowInvalidHostnames` to `true`
+to skip host name validation check for the server certificate.
+
+```json
+replication:
+  replSetName: rs1
+  syncSource: localhost:27017
+  ssl:
+    enabled: true
+    caFile: rootCA.pem
+```
+
+To create a self signed Certification Authority private key and certificate and a self signed Server private key and certificate:
+
+```
+openssl genrsa -out rootCA.key 2048
+openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.pem  -subj '/C=ES/ST=Spain/L=Madrid/O=8Kdata/CN=8Kdata'
+openssl x509 -in rootCA.pem -text -noout
+openssl genrsa -out mongodb-server.key
+openssl req -new -key mongodb-server.key -out mongodb-server.csr -subj '/C=ES/ST=Spain/L=Madrid/O=8Kdata/CN=localserver/DC=localserver'
+openssl x509 -req -in mongodb-server.csr -CA rootCA.pem -CAkey rootCA.key -CAcreateserial -out mongodb-server.crt -days 365 -sha256  -extensions san_env -extfile <(printf "[san_env]\nsubjectAltName=IP:127.0.0.1,DNS:localserver")
+openssl x509 -in mongodb-server.crt -text -noout
+cat mongodb-server.crt mongodb-server.key > mongodb-server.pem
+```
+
+If you prefer you may use a Java Key Store file that contain the same Certification Authority certificate:
+
+```json
+replication:
+  replSetName: rs1
+  syncSource: localhost:27017
+  ssl:
+    enabled: true
+    allowInvalidHostnames: true
+    trustStoreFile: rootCA.jks
+    trustStorePassword: trustme
+```
+
+To import the root certificate into a Java Key Store file:
+
+```
+openssl x509 -outform der -in rootCA.pem -out rootCA.der
+keytool -import -alias rootCA -keystore rootCA.jks -storepass trustme -trustcacerts -noprompt -file rootCA.der
+```
+
+## Authenticate against MongoDB
+
+If [MongoDB is configured to authenticate clients](https://docs.mongodb.com/manual/core/authentication-mechanisms/) you will have to create a user with role `__system` since ToroDB Stampede need to send special read only internal commands
+to the MongoDB Replica Set members.
+
+```
+db.getSiblingDB("admin").createUser({user: "stampede", pwd: "nosqlonsql", roles: [{role: "__system", db: "admin"}]})
+```
+
+You can then configure ToroDB Stampede to authenticate against MongoDB:
+
+```json
+replication:
+  replSetName: rs1
+  syncSource: localhost:27017
+  auth:
+    mode: negotiate
+    user: stampede
+    source: admin
+```
+
+You will need to add following entry to file `~/.mongopass` (or file specified by `/replication/mongopassFile` property):
+
+```
+localhost:27017:admin:stampede:nosqlonsql
+```
+
+### X.509 authentication
+
+If [MongoDB is configured to authenticate clients using certificates](https://docs.mongodb.com/manual/core/security-x.509/) you will have to create a client private key and certificate, 
+import them in a Java Key Store file and create a user with role `__system` in the `$external` database with name constructed by composing the certificate properties in a particular order (see below).
+
+To create the client private key and certificate.
+
+```
+openssl genrsa -out mongodb-client.key
+openssl req -new -key mongodb-client.key -out mongodb-client.csr -subj '/C=ES/ST=Spain/L=Madrid/O=8Kdata/CN=localclient/DC=localclient'
+openssl x509 -req -in mongodb-client.csr -CA rootCA.pem -CAkey rootCA.key -CAcreateserial -out mongodb-client.crt -days 365 -sha256  -extensions san_env -extfile <(printf "[san_env]\nsubjectAltName=IP:127.0.0.1,DNS:localclient")
+openssl x509 -in mongodb-client.crt -text -noout
+cat mongodb-client.crt mongodb-client.key > mongodb-client.pem
+```
+
+To create the Java Trust Store file `mongodb-client.jks` that contains the client certificate and private key starting from PEM files:
+
+```
+openssl pkcs12 -export -in mongodb-client.crt -inkey mongodb-client.key -out mongodb-client.p12 -name localclient -passout file:<(echo nosqlonsql)
+keytool -noprompt -importkeystore -deststorepass trustme -destkeypass nosqlonsql -destkeystore mongodb-client.jks -srckeystore mongodb-client.p12 -srcstoretype PKCS12 -srcstorepass nosqlonsql
+keytool -list -v -keystore mongodb-client.jks -storepass trustme -keypass nosqlonsql
+```
+
+To create the user that is authenticated with X.509 mechanism:
+
+```
+db.getSiblingDB("$external").createUser({user: "DC=localclient,CN=localclient,O=8Kdata,L=Madrid,ST=Spain,C=ES", roles: [{role: "__system", db: "admin"}]})
+```
+
+You can then configure ToroDB Stampede to authenticate against MongoDB:
+
+```json
+replication:
+  replSetName: rs1
+  syncSource: localhost:27017
+  auth:
+    mode: x509
+  ssl:
+    enabled: true
+    caFile: rootCA.pem
+    keyStoreFile: mongodb-client.jks
+    keyStorePassword: trustme
+    keyPassword: nosqlonsql
+```
+
+* `rootCA.pem` and `rootCA.key` should be the certificate and private key of the same Certification Authority used to generate the server certificate.
+
+In this case you do not have to add any entry to the file `~/.mongopass`.
diff --git a/documentation/docs/trouble-shooting.md b/documentation/docs/trouble-shooting.md
@@ -26,3 +26,37 @@ Once installed Homebrew, `wget` can be installed as follows:
 ```
 brew install wget
 ```
+
+## Duplication errors in the logs
+
+When Stampede is in recovery mode and, during the data importation, new data are inserted in the source, 
+ it's possible that Stampede reapplies the last batch of data. This would lead to error messages similar to:
+
+```
+  2017-06-21 16:43:03 CEST [21807-1] torodb@torod ERROR:  duplicate key value violates unique constraint "test__id_x_a_idx"
+  2017-06-21 16:43:03 CEST [21807-2] torodb@torod DETAIL:  Key (_id_x)=(\x594a858188b38a7816e4cfb9) already exists.
+  2017-06-21 16:43:03 CEST [21807-3] torodb@torod CONTEXT:  COPY test, line 1
+  2017-06-21 16:43:03 CEST [21807-4] torodb@torod STATEMENT:  COPY "test"."test" ("did","_id_x","x_d","a_s") FROM STDIN
+
+```
+
+  Fortunately, there is nothing to worry about this situation.
+
+## Unexpected optime errors
+
+Sometimes, the following error is shown:
+
+```
+
+Unexpected optime for last operation to apply. Expected {t: { "$timestamp": { "t": 1497464377, "i": 12} }, i: 30}, but {t: { "$timestamp": { "t": 1497464377, "i": 6} }, i: 30} found
+
+```
+
+This is due to the way in which the last applied operation time is calculated. 
+There is a comparison between a time which has been calculated taking into
+account all the operations in the oplog batch, and a time which only took
+into account replicated operations (that is, filtering out operations that 
+have been excluded by replication filters).
+
+So, when this log appears (DEBUG mode) is because the last operations of an
+ oplog batch are operations that are excluded by replication filters.
diff --git a/main/pom.xml b/main/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.torodb.stampede</groupId>
         <artifactId>stampede-pom</artifactId>
-        <version>1.0.0-beta3-SNAPSHOT</version>
+        <version>1.0.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>stampede-main</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -9,26 +9,26 @@
 
     <groupId>com.torodb.stampede</groupId>
     <artifactId>stampede-pom</artifactId>
-    <version>1.0.0-beta3-SNAPSHOT</version>
+    <version>1.0.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>ToroDB: Stampede</name>
     <description>ToroDB stampede project.</description>
-        
+
     <modules>
         <module>service</module>
         <module>main</module>
         <module>reporting</module>
     </modules>
-    
+
     <properties>
-        <torodb.engine.version>0.50.2-SNAPSHOT</torodb.engine.version>
-        
+        <torodb.engine.version>0.50.3-SNAPSHOT</torodb.engine.version>
+
         <license.header.license>agpl</license.header.license>
         <license.header.project.name>ToroDB Stampede</license.header.project.name>
         <license.header.project.inceptionYear>2016</license.header.project.inceptionYear>
     </properties>
-    
+
     <dependencyManagement>
         <dependencies>
             <dependency>
@@ -54,7 +54,7 @@
             </dependency>
         </dependencies>
     </dependencyManagement>
-    
+
     <build>
         <plugins>
             <plugin>
@@ -63,7 +63,7 @@
             </plugin>
         </plugins>
     </build>
-    
+
     <profiles>
         <profile>
             <id>docs</id>

diff --git a/reporting/pom.xml b/reporting/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.torodb.stampede</groupId>
         <artifactId>stampede-pom</artifactId>
-        <version>1.0.0-beta3-SNAPSHOT</version>
+        <version>1.0.0-SNAPSHOT</version>
     </parent>
     <artifactId>reporting</artifactId>
     <name>ToroDB: Stampede reporting</name>

diff --git a/service/pom.xml b/service/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.torodb.stampede</groupId>
         <artifactId>stampede-pom</artifactId>
-        <version>1.0.0-beta3-SNAPSHOT</version>
+        <version>1.0.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>stampede-service</artifactId>