forked from Countly/countly-server
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
691a8c4
commit 46d436d
Showing
4 changed files
with
170 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Export all mongodb data using collection by collection export | ||
| # Server: mongodb or any server with mongotools installed with mongoexport command | ||
| # Path: any | ||
| # Command: bash 1_full_export.sh | ||
|
|
||
| #connection string without database | ||
| connection_string="mongodb://localhost" | ||
|
|
||
| #database which to export, countly has data in 2 database countly and countly_drill | ||
| db="countly" | ||
|
|
||
| #output where to store the exported files | ||
| out_dir="./output" | ||
|
|
||
|
|
||
| if [ ! $out_dir ]; then | ||
| out_dir="./" | ||
| else | ||
| mkdir -p $out_dir | ||
| fi | ||
|
|
||
| tmp_file="fadlfhsdofheinwvw.js" | ||
| echo "print('_ ' + db.getCollectionNames())" > $tmp_file | ||
| cols=`mongo "${connection_string}/${db}" $tmp_file | grep '_' | awk '{print $2}' | tr ',' ' '` | ||
| for c in $cols | ||
| do | ||
| mongoexport --uri="${connection_string}" -d $db -c $c -o "$out_dir/${db}_${c}.json" | ||
| done | ||
| rm $tmp_file |
55 changes: 55 additions & 0 deletions
55
bin/scripts/export-data/2_full_export_specific_collections.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| #!/bin/bash | ||
|
|
||
|
|
||
| # Export all mongodb data for some specific collections with prefix | ||
| # Server: mongodb or any server with mongotools installed with mongoexport command | ||
| # Path: any | ||
| # Command: bash 2_full_export_specific_collections.sh | ||
|
|
||
|
|
||
| #connection string without database | ||
| connection_string="mongodb://localhost" | ||
|
|
||
| #prefix of collection | ||
| prefix="app_users" | ||
|
|
||
| #database which to export, countly has data in 2 database countly and countly_drill | ||
| db="countly" | ||
|
|
||
| #output where to store the exported files | ||
| out_dir="./output" | ||
|
|
||
| if [ ! $out_dir ]; then | ||
| out_dir="./" | ||
| else | ||
| mkdir -p $out_dir | ||
| fi | ||
|
|
||
| tmp_file="fadlfhsdofheinwvw.js" | ||
| echo "print('_ ' + db.getCollectionNames().filter(function(c){return c.indexOf(\"${prefix}\") === 0}))" > $tmp_file | ||
| cols=`mongo "${connection_string}/${db}" $tmp_file | grep '_' | awk '{print $2}' | tr ',' ' '` | ||
| for c in $cols | ||
| do | ||
| mongoexport --uri="${connection_string}" -d $db -c $c -o "$out_dir/${db}_${c}.json" | ||
| done | ||
| rm $tmp_file | ||
|
|
||
|
|
||
| prefix="drill_events" | ||
| db="countly_drill" | ||
| out_dir="./output" | ||
|
|
||
| if [ ! $out_dir ]; then | ||
| out_dir="./" | ||
| else | ||
| mkdir -p $out_dir | ||
| fi | ||
|
|
||
| tmp_file="fadlfhsdofheinwvw.js" | ||
| echo "print('_ ' + db.getCollectionNames().filter(function(c){return c.indexOf(\"${prefix}\") === 0}))" > $tmp_file | ||
| cols=`mongo "${connection_string}/${db}" $tmp_file | grep '_' | awk '{print $2}' | tr ',' ' '` | ||
| for c in $cols | ||
| do | ||
| mongoexport --uri="${connection_string}" -d $db -c $c -o "$out_dir/${db}_${c}.json" | ||
| done | ||
| rm $tmp_file |
58 changes: 58 additions & 0 deletions
58
bin/scripts/export-data/3_incremental_export_specific_collections.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Incremental export, that can be run periodically to export only new data. Exports latest updated users and new drill events | ||
| # Server: mongodb or any server with mongotools installed with mongoexport command | ||
| # Path: any | ||
| # Command: bash 3_incremental_export_specific_collections.sh | ||
|
|
||
| #connection string without database | ||
| connection_string="mongodb://localhost" | ||
|
|
||
| #how many seconds back data should be exported, for example if you want to export data for last 24 horus, it would be 86400 seconds (24*60*60) | ||
| seconds=86400 | ||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
| timestamp="$(date +"%s")" | ||
| start_timestamp=$((${timestamp}-${seconds})) | ||
| start_date="$(date -u +'%Y-%m-%dT%H:%M:%S+0000' -d @${start_timestamp})" | ||
| prefix="app_users" | ||
| db="countly" | ||
| out_dir="./output" | ||
|
|
||
| if [ ! $out_dir ]; then | ||
| out_dir="./" | ||
| else | ||
| mkdir -p $out_dir | ||
| fi | ||
|
|
||
| tmp_file="fadlfhsdofheinwvw.js" | ||
| echo "print('_ ' + db.getCollectionNames().filter(function(c){return c.indexOf(\"${prefix}\") === 0}))" > $tmp_file | ||
| cols=`mongo "${connection_string}/${db}" $tmp_file | grep '_' | awk '{print $2}' | tr ',' ' '` | ||
| for c in $cols | ||
| do | ||
| mongoexport --uri="${connection_string}" -d $db -c $c -o "$out_dir/${db}_${c}.json" --query "{\"last_sync\":{\"\$gte\":${start_timestamp}}}" | ||
| done | ||
| rm $tmp_file | ||
|
|
||
|
|
||
| prefix="drill_events" | ||
| db="countly_drill" | ||
| out_dir="./output" | ||
|
|
||
| if [ ! $out_dir ]; then | ||
| out_dir="./" | ||
| else | ||
| mkdir -p $out_dir | ||
| fi | ||
|
|
||
| tmp_file="fadlfhsdofheinwvw.js" | ||
| echo "print('_ ' + db.getCollectionNames().filter(function(c){return c.indexOf(\"${prefix}\") === 0}))" > $tmp_file | ||
| cols=`mongo "${connection_string}/${db}" $tmp_file | grep '_' | awk '{print $2}' | tr ',' ' '` | ||
| for c in $cols | ||
| do | ||
| mongoexport --uri="${connection_string}" -d $db -c $c -o "$out_dir/${db}_${c}.json" --query "{\"cd\":{\"\$gte\":{\"\$date\": \"${start_date}\"}}}" | ||
| done | ||
| rm $tmp_file |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Creates needed indexes for incremental exports | ||
| # Server: mongodb or any server with mongotools installed with mongoexport command | ||
| # Path: any | ||
| # Command: bash 4_create_incremental_indexes.sh | ||
|
|
||
| #connection string without database | ||
| connection_string="mongodb://localhost" | ||
|
|
||
|
|
||
|
|
||
| prefix="app_users" | ||
| db="countly" | ||
| tmp_file="fadlfhsdofheinwvw.js" | ||
| echo "db.getCollectionNames().forEach(function(c){if(c.indexOf(\"${prefix}\") === 0){db[c].createIndex({last_sync: -1},{background: true})}})" > $tmp_file | ||
| cols=`mongo "${connection_string}/${db}" $tmp_file` | ||
| rm $tmp_file | ||
|
|
||
|
|
||
| prefix="drill_events" | ||
| db="countly_drill" | ||
| tmp_file="fadlfhsdofheinwvw.js" | ||
| echo "db.getCollectionNames().forEach(function(c){if(c.indexOf(\"${prefix}\") === 0){db[c].createIndex({cd: -1},{background: true})}})" > $tmp_file | ||
| cols=`mongo "${connection_string}/${db}" $tmp_file` | ||
| rm $tmp_file |