Skip to content

Commit

Permalink
Possible fix for mongodb-raw find all docs
Browse files Browse the repository at this point in the history
  • Loading branch information
sumeetchhetri committed May 5, 2024
1 parent 6609e94 commit e740f12
Show file tree
Hide file tree
Showing 7 changed files with 235 additions and 10 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci-win-mingw.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ jobs:
pacman -S --noconfirm mingw-w64-x86_64-hiredis
pacman -S --noconfirm diffutils
pacman -S --noconfirm unzip
pacman -S --noconfirm dos2unix
echo 'Installed all dependencies'
- name: Install dependencies and ffead-cpp
run: |
Expand Down
10 changes: 6 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -858,10 +858,12 @@ if(SRV_ALL)
set_property(TARGET ffead-cpp-drogon PROPERTY POSITION_INDEPENDENT_CODE ON)
target_link_libraries(ffead-cpp-drogon ffead-modules ffead-framework -Wl,--copy-dt-needed-entries ${HAVE_DROGONLIB} ${HAVE_TRANTORLIB} ${HAVE_JSONCPP} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_ELASTICLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB} brotlidec brotlienc)

add_executable(ffead-cpp-nghttp2 src/server/nghttp2/Nghttp2Server.cpp)
find_package(Boost 1.65 REQUIRED COMPONENTS system thread)
set_property(TARGET ffead-cpp-nghttp2 PROPERTY POSITION_INDEPENDENT_CODE ON)
target_link_libraries(ffead-cpp-nghttp2 ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_NGHTTP2ASIOLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_ELASTICLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB})
if(NOT DEBUG)
add_executable(ffead-cpp-nghttp2 src/server/nghttp2/Nghttp2Server.cpp)
find_package(Boost 1.65 REQUIRED COMPONENTS system thread)
set_property(TARGET ffead-cpp-nghttp2 PROPERTY POSITION_INDEPENDENT_CODE ON)
target_link_libraries(ffead-cpp-nghttp2 ffead-modules ffead-framework ${Boost_LIBRARIES} ${HAVE_NGHTTP2ASIOLIB} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${HAVE_ONIG_REGEX_LIB} ${HAVE_UUIDLIB} ${HAVE_CURLLIB} ${HAVE_SSLLIB} ${HAVE_REDISLIB} ${HAVE_REDIS_CLUSTERLIB} ${HAVE_MEMCACHEDLIB} ${HAVE_MEMCACHEDUTILLIB} ${HAVE_ODBCLIB} ${HAVE_ELASTICLIB} ${HAVE_MONGOCLIB} ${HAVE_SCYLLALIB} ${HAVE_PUGI_XML_LIB} ${HAVE_BSONLIB} ${HAVE_ZLIB} ${HAVE_CRYPTOLIB})
endif()

#add_executable(ffead-cpp-mongols src/server/mongols/MongolsServer.cpp)
#set_property(TARGET ffead-cpp-mongols PROPERTY POSITION_INDEPENDENT_CODE ON)
Expand Down
4 changes: 2 additions & 2 deletions docker/all/base/ffead-cpp-all-base-debug.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selectio

WORKDIR ${IROOT}

COPY install_ffead-cpp-framework.sh install_ffead-cpp-httpd.sh install_ffead-cpp-nginx.sh server.sh ${IROOT}/
COPY install_ffead-cpp-framework-debug.sh install_ffead-cpp-httpd.sh install_ffead-cpp-nginx.sh server.sh ${IROOT}/
RUN chmod 755 ${IROOT}/*.sh
RUN ./install_ffead-cpp-framework.sh && ./install_ffead-cpp-httpd.sh && ./install_ffead-cpp-nginx.sh && cd ${IROOT}/ffead-cpp-src && ninja clean && rm -rf CMakeFiles CMakeCache.txt
RUN ./install_ffead-cpp-framework-debug.sh && cd ${IROOT}/ffead-cpp-src && ninja clean && rm -rf CMakeFiles CMakeCache.txt

COPY run_ffead.sh /
RUN chmod 755 /run_ffead.sh
Expand Down
190 changes: 190 additions & 0 deletions docker/all/scripts/install_ffead-cpp-framework-debug.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
#!/bin/bash

#Set the number of threads ----
MAX_THREADS=$(( 3 * `nproc` / 2 ))

WRIT_THREADS=$(( $MAX_THREADS / 3 ))
SERV_THREADS=$(( $MAX_THREADS - $WRIT_THREADS ))

#temporary workaround till this gets fixed by the framework author of CppServer
sed -i 's|explicit constexpr UUID(const char\* uuid, size_t size);|explicit UUID(const char\* uuid, size_t size);|g' /usr/local/include/system/uuid.h
sed -i 's|constexpr CppCommon::UUID operator ""|CppCommon::UUID operator ""|g' /usr/local/include/system/uuid.h
sed -i 's|inline constexpr UUID::UUID|inline UUID::UUID|g' /usr/local/include/system/uuid.inl

git clone https://github.com/sumeetchhetri/ffead-cpp
#git checkout 92c3a9e3d5ec1de4a909fe688d649d7f31e050c0 -b 6.0
cd ffead-cpp
rm -rf .git
cd ..
mv ffead-cpp ffead-cpp-src
mv ffead-cpp-src/lang-server-backends ${IROOT}/

cd /tmp
git clone https://github.com/Tencent/rapidjson
cp -rf rapidjson/include/rapidjson /usr/include/
rm -rf rapidjson

cd $IROOT/ffead-cpp-src/

chmod 755 *.sh resources/*.sh rtdcf/autotools/*.sh
#rm -rf web/t1
#rm -rf web/t2
#rm -rf web/t3
#rm -rf web/t4
#rm -rf web/t5
#rm -rf web/t6
#rm -rf web/t7
mv ${IROOT}/server.sh script/
#mv ${IROOT}/t1 web/
#mv ${IROOT}/t2 web/
#mv ${IROOT}/t3 web/
#mv ${IROOT}/t4 web/
#mv ${IROOT}/t5 web/
#mv ${IROOT}/t6 web/
#mv ${IROOT}/t7 web/
sed -i 's|THRD_PSIZ=6|THRD_PSIZ='${SERV_THREADS}'|g' resources/server.prop
sed -i 's|W_THRD_PSIZ=2|W_THRD_PSIZ='${WRIT_THREADS}'|g' resources/server.prop
sed -i 's|ENABLE_CRS=true|ENABLE_CRS=false|g' resources/server.prop
sed -i 's|ENABLE_SEC=true|ENABLE_SEC=false|g' resources/server.prop
sed -i 's|ENABLE_FLT=true|ENABLE_FLT=false|g' resources/server.prop
sed -i 's|ENABLE_CNT=true|ENABLE_CNT=true|g' resources/server.prop
sed -i 's|ENABLE_EXT_CNT=true|ENABLE_EXT_CNT=false|g' resources/server.prop
sed -i 's|ENABLE_CNT_MPG=true|ENABLE_CNT_MPG=false|g' resources/server.prop
sed -i 's|ENABLE_CNT_PTH=true|ENABLE_CNT_PTH=false|g' resources/server.prop
sed -i 's|ENABLE_CNT_EXT=true|ENABLE_CNT_EXT=false|g' resources/server.prop
sed -i 's|ENABLE_CNT_RST=true|ENABLE_CNT_RST=true|g' resources/server.prop
sed -i 's|ENABLE_EXT=true|ENABLE_EXT=true|g' resources/server.prop
sed -i 's|ENABLE_SCR=true|ENABLE_SCR=false|g' resources/server.prop
sed -i 's|ENABLE_SWS=true|ENABLE_SWS=false|g' resources/server.prop
sed -i 's|ENABLE_JOBS=true|ENABLE_JOBS=false|g' resources/server.prop
sed -i 's|LOGGING_ENABLED=true|LOGGING_ENABLED=false|g' resources/server.prop
sed -i 's|EVH_SINGLE=true|EVH_SINGLE=false|g' resources/server.prop

rm -rf web/default web/oauthApp web/flexApp web/markers web/te-benchmark web/peer-server
#rm -rf web/t1 web/t2

sed -i 's|localhost|tfb-database|g' web/t1/config/sdorm.xml
sed -i 's|localhost|tfb-database|g' web/t2/config/sdorm.xml
sed -i 's|localhost|tfb-database|g' web/t3/config/sdorm.xml
sed -i 's|localhost|tfb-database|g' web/t4/config/sdorm.xml
sed -i 's|localhost|tfb-database|g' web/t5/config/sdorm.xml
sed -i 's|localhost|tfb-database|g' web/t6/config/sdorm.xml
sed -i 's|localhost|tfb-database|g' web/t7/config/sdorm.xml
sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/default)||g' CMakeLists.txt
sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/flexApp)||g' CMakeLists.txt
sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/oauthApp)||g' CMakeLists.txt
sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/markers)||g' CMakeLists.txt
sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/te-benchmark)||g' CMakeLists.txt
sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/peer-server)||g' CMakeLists.txt
#sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/t1)||g' CMakeLists.txt
#sed -i 's|add_subdirectory(${PROJECT_SOURCE_DIR}/web/t2)||g' CMakeLists.txt
sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/default/libdefault${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/flexApp/libflexApp${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/oauthApp/liboauthApp${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/markers/libmarkers${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/te-benchmark/libte-benchmark${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/peer-server/libpeer-server${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
#sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/t1/libt1${LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
#sed -i 's|install(FILES ${PROJECT_BINARY_DIR}/web/t2/libt2{LIB_EXT} DESTINATION ${PROJECT_NAME}-bin/lib)||g' CMakeLists.txt
sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/default")||g' CMakeLists.txt
sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/flexApp")||g' CMakeLists.txt
sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/oauthApp")||g' CMakeLists.txt
sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/markers")||g' CMakeLists.txt
sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/peer-server")||g' CMakeLists.txt
sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/te-benchmark")||g' CMakeLists.txt
#sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/t1")||g' CMakeLists.txt
#sed -i 's|install(DIRECTORY DESTINATION "${PROJECT_NAME}-bin/web/t2")||g' CMakeLists.txt
sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/default/ DESTINATION ${PROJECT_NAME}-bin/web/default)||g' CMakeLists.txt
sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/flexApp/ DESTINATION ${PROJECT_NAME}-bin/web/flexApp)||g' CMakeLists.txt
sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/oauthApp/ DESTINATION ${PROJECT_NAME}-bin/web/oauthApp)||g' CMakeLists.txt
sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/markers/ DESTINATION ${PROJECT_NAME}-bin/web/markers)||g' CMakeLists.txt
sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/peer-server/ DESTINATION ${PROJECT_NAME}-bin/web/peer-server)||g' CMakeLists.txt
sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/te-benchmark/ DESTINATION ${PROJECT_NAME}-bin/web/te-benchmark)||g' CMakeLists.txt
#sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/t1/ DESTINATION ${PROJECT_NAME}-bin/web/t1)||g' CMakeLists.txt
#sed -i 's|install(DIRECTORY ${PROJECT_SOURCE_DIR}/web/t2/ DESTINATION ${PROJECT_NAME}-bin/web/t2)||g' CMakeLists.txt
sed -i 's|web/default/src/autotools/Makefile||g' configure.ac
sed -i 's|web/flexApp/src/autotools/Makefile||g' configure.ac
sed -i 's|web/oauthApp/src/autotools/Makefile||g' configure.ac
sed -i 's|web/markers/src/autotools/Makefile||g' configure.ac
sed -i 's|web/te-benchmark/src/autotools/Makefile||g' configure.ac
sed -i 's|web/peer-server/src/autotools/Makefile||g' configure.ac
#sed -i 's|web/t1/src/autotools/Makefile||g' configure.ac
#sed -i 's|web/t2/src/autotools/Makefile||g' configure.ac

#./autogen.sh
#./configure --enable-debug=no --enable-apachemod=yes --enable-nginxmod=yes --enable-mod_sdormmongo=yes --enable-mod_sdormsql=yes --enable-mod_rediscache=yes --enable-mod_memcached=yes CPPFLAGS="$CPPFLAGS -I${IROOT}/include/libmongoc-1.0 -I${IROOT}/include/libbson-1.0 -I${IROOT}/include/" LDFLAGS="$LDFLAGS -L${IROOT} -L${IROOT}/lib"
cmake -DSRV_EMB=on -DCINATRA_INCLUDES=${IROOT}/cinatra/include -DMOD_APACHE=on -DMOD_NGINX=on -DMOD_MEMCACHED=on -DMOD_REDIS=on -DMOD_SDORM_MONGO=on -DDEBUG=${DEBUG} -DWITH_RAPIDJSON=on -DWITH_PUGIXML=on -GNinja .

cp resources/sample-odbcinst.ini ${IROOT}/odbcinst.ini
cp resources/sample-odbc.ini ${IROOT}/odbc.ini

sed -i 's|127.0.0.1|tfb-database|g' ${IROOT}/odbc.ini

#Start building for mongodb as the World model is different for SQL use case
cd ${IROOT}/ffead-cpp-src/
cp -f web/t1/sql-src/TeBkUmWorldmongo.h web/t1/include/TeBkUmWorld.h
cp -f web/t1/sql-src/TeBkUmWorldmongo.cpp web/t1/src/TeBkUmWorld.cpp
ninja install

rm -f /usr/local/lib/libffead-*
rm -f /usr/local/lib/libt1.so*
rm -f /usr/local/lib/libt2.so*
rm -f /usr/local/lib/libt3.so*
rm -f /usr/local/lib/libt4.so*
rm -f /usr/local/lib/libt5.so*
rm -f /usr/local/lib/libt6.so*
rm -f /usr/local/lib/libt7.so*
rm -f /usr/local/lib/libinter.so
rm -f /usr/local/lib/libdinter.so

if [ ! -d "ffead-cpp-7.0-bin" ]
then
exit 1
fi

cd ffead-cpp-7.0-bin
#cache related dockerfiles will add the cache.xml accordingly whenever needed
chmod 755 *.sh resources/*.sh rtdcf/autotools/*.sh
./server.sh &
COUNTER=0
while [ ! -f lib/libinter.so ]
do
sleep 1
COUNTER=$((COUNTER+1))
if [ "$COUNTER" = 120 ]
then
cat logs/jobs.log
echo "ffead-cpp exiting exiting due to failure...."
exit 1
fi
done
COUNTER=0
while [ ! -f lib/libdinter.so ]
do
sleep 1
COUNTER=$((COUNTER+1))
if [ "$COUNTER" = 120 ]
then
cat logs/jobs.log
echo "ffead-cpp exiting exiting due to failure....ddlib"
exit 1
fi
done
echo "ffead-cpp start successful"
sleep 20
cd tests && rm -f test.csv && cp ${IROOT}/ffead-cpp-src/tests/test-te-all.csv test.csv && chmod +x *.sh && ./runTests.sh
echo "ffead-cpp normal shutdown"
pkill ffead-cpp

cd ${IROOT}/ffead-cpp-src/
cp -rf ffead-cpp-7.0-bin ${IROOT}/ffead-cpp-7.0
rm -rf ffead-cpp-7.0-bin
mv ${IROOT}/nginxfc ${IROOT}/nginx-ffead-mongo

cd ${IROOT}/ffead-cpp-7.0

chmod 755 *.sh resources/*.sh rtdcf/autotools/*.sh
chmod 755 *.sh
rm -f *.cntrl
rm -f tmp/*.sess
#Done building for mongodb
32 changes: 32 additions & 0 deletions docker/build-all-te-debug-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
####As some images use "-march=native -mtune=native" we need to build these below images locally and push to docker
VERSION=""

if [ ! -z "$1" ]
then
VERSION=$1
fi

if [ -z "$VERSION" ]
then
echo 'Version cannot be blank'
exit 0
fi

cd all
rm -rf temp
mkdir temp
cp base/*.dockerfile scripts/*.sh *.sh *.dockerfile temp/
#cp -rf postgresql temp/
cd temp

find . -type f -name '*.dockerfile' | xargs sed -i'' -e "s|-deps:6.0|-deps:${VERSION}|g"
find . -type f -name '*.dockerfile' | xargs sed -i'' -e "s|-base:6.0|-base:${VERSION}|g"
find . -type f -name '*.dockerfile' | xargs sed -i'' -e "s|-base-2:6.0|-base-2:${VERSION}|g"
find . -type f -name '*.dockerfile' | xargs sed -i'' -e "s|version=\"6.0\"|version=\"${VERSION}\"|g"

docker rmi -f sumeetchhetri/ffead-cpp-base:${VERSION}-debug
#docker buildx create --use --name insecbuild --buildkitd-flags '--allow-insecure-entitlement security.insecure'
#docker buildx build --ulimit memlock=102400000:102400000 --allow security.insecure --progress plain -f ffead-cpp-all-base.dockerfile -t sumeetchhetri/ffead-cpp-base:${VERSION} .
docker build --ulimit memlock=102400000:102400000 --progress plain -f ffead-cpp-all-base-debug.dockerfile -t sumeetchhetri/ffead-cpp-base:${VERSION}-debug .
#docker buildx rm --all-inactive --force
docker push sumeetchhetri/ffead-cpp-base:${VERSION}-debug
4 changes: 2 additions & 2 deletions src/modules/sdorm/mongo/raw/MongoDBRawDataSourceImpl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ void MongoDBRawDataSourceImpl::executeQuery(const std::string &query, void* ctx,
bson_destroy(qb);
}

void MongoDBRawDataSourceImpl::executeQuery(const bson_t* query, void* ctx, MgRawResFunc cb, int start, int count) {
void MongoDBRawDataSourceImpl::executeQuery(bson_t* query, void* ctx, MgRawResFunc cb, int start, int count) {
bool qi = false;
if(query==NULL) {
qi = true;
Expand Down Expand Up @@ -272,7 +272,7 @@ bool MongoDBRawDataSourceImpl::executeUpdateQuery(const std::string &query, cons
return fl;
}

bool MongoDBRawDataSourceImpl::executeUpdateQuery(const bson_t* query, const bson_t* data, MongoDBRawOper oper) {
bool MongoDBRawDataSourceImpl::executeUpdateQuery(bson_t* query, bson_t* data, MongoDBRawOper oper) {
bson_error_t er;
switch(oper) {
case INSERT:
Expand Down
4 changes: 2 additions & 2 deletions src/modules/sdorm/mongo/raw/MongoDBRawDataSourceImpl.h
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ class MongoDBRawDataSourceImpl : public DataSourceType {
void addBulk(bson_t* q, bson_t* d);
bool endBulk();

void executeQuery(const bson_t* query, void* ctx, MgRawResFunc cb, int start = 0, int count = 0);
void executeQuery(bson_t* query, void* ctx, MgRawResFunc cb, int start = 0, int count = 0);
void executeQuery(const std::string &query, void* ctx, MgRawResFunc cb, int start = 0, int count = 0);
bool executeUpdateQuery(const std::string &query, const std::string &data, MongoDBRawOper oper);
bool executeUpdateQuery(const bson_t* query, const bson_t* data, MongoDBRawOper oper);
bool executeUpdateQuery(bson_t* query, bson_t* data, MongoDBRawOper oper);
bool executeUpdateBulkQuery(int len, MgRawBlkFunc cb);
};

Expand Down

0 comments on commit e740f12

Please sign in to comment.