Merge remote-tracking branch 'origin/devel' into feature/distribute-s…

…implification # Conflicts: # arangod/Aql/AqlFunctionFeature.cpp
arangodb · Feb 4, 2021 · b6269be · b6269be
2 parents d5b2ac8 + 2893749
commit b6269be
Show file tree

Hide file tree

Showing 19 changed files with 246 additions and 178 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,15 @@
 devel
 -----
 
+* Remove logging startup options `--log.api-enabled` and `--log.keep-logrotate` 
+  for all client tools (arangosh, arangodump, arangorestore etc.), as these 
+  options are only meaningful for arangod.
+
+* Fixed BTS-284: upgrading from 3.6 to 3.7 in cluster enviroment.
+  Moved upgrade ArangoSearch links task to later step as it needs cluster
+  connection. Removed misleading error log records for failed ArangoSearch index
+  creation during upgrade phase.
+
 * Extend the "move-calculations-up" optimizer rule so that it can move
   calculations out of subqueries into the outer query.
 

diff --git a/arangod/Aql/AqlFunctionFeature.cpp b/arangod/Aql/AqlFunctionFeature.cpp
@@ -47,12 +47,6 @@ AqlFunctionFeature::AqlFunctionFeature(application_features::ApplicationServer&
   startsAfter<AqlFeature>();
 }
 
-// This feature does not have any options
-void AqlFunctionFeature::collectOptions(std::shared_ptr<options::ProgramOptions>) {}
-
-// This feature does not have any options
-void AqlFunctionFeature::validateOptions(std::shared_ptr<options::ProgramOptions>) {}
-
 void AqlFunctionFeature::prepare() {
   /// @brief Add all AQL functions to the FunctionDefintions map
   addTypeCheckFunctions();
@@ -88,6 +82,10 @@ void AqlFunctionFeature::addAlias(std::string const& alias, std::string const& o
 void AqlFunctionFeature::toVelocyPack(VPackBuilder& builder) const {
   builder.openArray();
   for (auto const& it : _functionNames) {
+    if (it.second.hasFlag(FF::Internal)) {
+      // don't serialize internal functions
+      continue;
+    }
     it.second.toVelocyPack(builder);
   }
   builder.close();
@@ -487,6 +485,11 @@ void AqlFunctionFeature::addMiscFunctions() {
        Function::makeFlags(FF::Deterministic, FF::Cacheable,
                            FF::CanRunOnDBServerCluster, FF::CanRunOnDBServerOneShard),
        &Functions::MakeDistributeGraphInput});
+
+  // this is an internal function that is only here for testing. it cannot
+  // be invoked by end users, because refering to internal functions from user
+  // queries will pretend these functions do not exist.
+  add({"INTERNAL", "", Function::makeFlags(FF::Internal), &Functions::NotImplemented});
 }
 
 }  // namespace aql

diff --git a/arangod/Aql/AqlFunctionFeature.h b/arangod/Aql/AqlFunctionFeature.h
@@ -39,8 +39,6 @@ class AqlFunctionFeature final : public application_features::ApplicationFeature
  public:
   explicit AqlFunctionFeature(application_features::ApplicationServer& server);
 
-  void collectOptions(std::shared_ptr<options::ProgramOptions>) override final;
-  void validateOptions(std::shared_ptr<options::ProgramOptions>) override final;
   void prepare() override final;
 
   void add(Function const& func);

diff --git a/arangod/Aql/Ast.cpp b/arangod/Aql/Ast.cpp
@@ -1583,7 +1583,7 @@ AstNode* Ast::createNodeAggregateFunctionCall(char const* functionName, AstNode
 
 /// @brief create an AST function call node
 AstNode* Ast::createNodeFunctionCall(char const* functionName, size_t length,
-                                     AstNode const* arguments) {
+                                     AstNode const* arguments, bool allowInternalFunctions) {
   if (functionName == nullptr) {
     THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
   }
@@ -1597,6 +1597,14 @@ AstNode* Ast::createNodeFunctionCall(char const* functionName, size_t length,
     auto& server = query().vocbase().server();
     auto func = server.getFeature<AqlFunctionFeature>().byName(normalized.first);
     TRI_ASSERT(func != nullptr);
+
+    if (!allowInternalFunctions && func->hasFlag(Function::Flags::Internal)) {
+      // a function flagged as internal, but internal functions cannot be used in this context.
+      // throw an error pretending that the function does not exist
+      std::string msg = basics::Exception::FillExceptionString(TRI_ERROR_QUERY_FUNCTION_NAME_UNKNOWN, normalized.first.c_str());
+      msg.append(" - this is an internal function and not supposed to be used directly");
+      THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_FUNCTION_NAME_UNKNOWN, std::move(msg));
+    }
 
     node = createNode(NODE_TYPE_FCALL);
     // register a pointer to the function
@@ -1618,7 +1626,7 @@ AstNode* Ast::createNodeFunctionCall(char const* functionName, size_t length,
                                     static_cast<int>(numExpectedArguments.first),
                                     static_cast<int>(numExpectedArguments.second));
     }
-
+    
     if (func->hasFlag(Function::Flags::CanReadDocuments)) {
       // this also qualifies a query for potentially reading documents via function calls!
       _functionsMayAccessDocuments = true;
@@ -1638,6 +1646,11 @@ AstNode* Ast::createNodeFunctionCall(char const* functionName, size_t length,
   return node;
 }
 
+AstNode* Ast::createNodeFunctionCall(char const* functionName, AstNode const* arguments,
+                                     bool allowInternalFunctions) {
+  return createNodeFunctionCall(functionName, strlen(functionName), arguments, allowInternalFunctions);
+}
+
 /// @brief create an AST range node
 AstNode* Ast::createNodeRange(AstNode const* start, AstNode const* end) {
   AstNode* node = createNode(NODE_TYPE_RANGE);
@@ -3438,7 +3451,7 @@ AstNode* Ast::optimizeFunctionCall(transaction::Methods& trx,
         auto countArgs = createNodeArray();
         countArgs->addMember(createNodeValueString(arg->getStringValue(),
                                                    arg->getStringLength()));
-        return createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("COLLECTION_COUNT"), countArgs);
+        return createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("COLLECTION_COUNT"), countArgs, true);
       }
     }
   } else if (func->name == "IS_NULL") {
@@ -4169,7 +4182,3 @@ AstNode* Ast::createNodeAttributeAccess(AstNode const* node,
                  [](basics::AttributeName const& a) { return a.name; });
   return createNodeAttributeAccess(node, vec);
 }
-
-AstNode* Ast::createNodeFunctionCall(char const* functionName, AstNode const* arguments) {
-  return createNodeFunctionCall(functionName, strlen(functionName), arguments);
-}
diff --git a/arangod/Aql/Ast.h b/arangod/Aql/Ast.h
@@ -380,10 +380,11 @@ class Ast {
   AstNode* createNodeKShortestPaths(arangodb::graph::ShortestPathType::Type type, AstNode const*, AstNode const*);
 
   /// @brief create an AST function call node
-  AstNode* createNodeFunctionCall(char const* functionName, AstNode const* arguments);
+  AstNode* createNodeFunctionCall(char const* functionName, AstNode const* arguments,
+                                  bool allowInternalFunctions);
 
   AstNode* createNodeFunctionCall(char const* functionName, size_t length,
-                                  AstNode const* arguments);
+                                  AstNode const* arguments, bool allowInternalFunctions);
 
   /// @brief create an AST function call node for aggregate functions
   AstNode* createNodeAggregateFunctionCall(char const* functionName, AstNode const* arguments);

diff --git a/arangod/Aql/Function.h b/arangod/Aql/Function.h
@@ -80,6 +80,11 @@ struct Function {
     /// @brief exclude the function from being evaluated during AST
     /// optimizations evaluation of function will only happen at query runtime
     NoEval = 32,
+
+    /// @brief internal function, supposed to be inserted only by internal
+    /// optimizations and transformations. not supposed to be used by end users
+    /// and thus not documented in the official list of available AQL functions.
+    Internal = 64,
   };
 
   /// @brief helper for building flags

diff --git a/arangod/Aql/IndexNode.cpp b/arangod/Aql/IndexNode.cpp
@@ -265,10 +265,10 @@ arangodb::aql::AstNode* IndexNode::makeUnique(arangodb::aql::AstNode* node) cons
     if (idx->sparse() || idx->isSorted()) {
       // the index is sorted. we need to use SORTED_UNIQUE to get the
       // result back in index order
-      return ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("SORTED_UNIQUE"), array);
+      return ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("SORTED_UNIQUE"), array, true);
     }
     // a regular UNIQUE will do
-    return ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("UNIQUE"), array);
+    return ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("UNIQUE"), array, true);
   }
 
   // presumably an array with no or a single member

diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp
@@ -896,7 +896,7 @@ void arangodb::aql::sortInValuesRule(Optimizer* opt, std::unique_ptr<ExecutionPl
       auto args = ast->createNodeArray();
       args->addMember(rhs);
       auto sorted =
-          ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("SORTED_UNIQUE"), args);
+          ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("SORTED_UNIQUE"), args, true);
       inNode->changeMember(1, sorted);
       modified = true;
       continue;
@@ -990,7 +990,7 @@ void arangodb::aql::sortInValuesRule(Optimizer* opt, std::unique_ptr<ExecutionPl
     auto args = ast->createNodeArray();
     args->addMember(originalArg);
     auto sorted =
-        ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("SORTED_UNIQUE"), args);
+        ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("SORTED_UNIQUE"), args, true);
 
     auto outVar = ast->variables()->createTemporaryVariable();
     auto expression = std::make_unique<Expression>(ast, sorted);
@@ -7043,7 +7043,7 @@ static std::unique_ptr<Condition> buildGeoCondition(ExecutionPlan* plan,
 
     addLocationArg(args);
     AstNode* func =
-        ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("GEO_DISTANCE"), args);
+        ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("GEO_DISTANCE"), args, true);
 
     TRI_ASSERT(info.maxDistanceExpr || info.minDistanceExpr || info.sorted);
     if (info.minDistanceExpr != nullptr) {
@@ -7073,9 +7073,9 @@ static std::unique_ptr<Condition> buildGeoCondition(ExecutionPlan* plan,
     args->addMember(info.filterExpr);
     addLocationArg(args);
     if (info.filterMode == geo::FilterType::CONTAINS) {
-      cond->andCombine(ast->createNodeFunctionCall("GEO_CONTAINS", args));
+      cond->andCombine(ast->createNodeFunctionCall("GEO_CONTAINS", args, true));
     } else if (info.filterMode == geo::FilterType::INTERSECTS) {
-      cond->andCombine(ast->createNodeFunctionCall("GEO_INTERSECTS", args));
+      cond->andCombine(ast->createNodeFunctionCall("GEO_INTERSECTS", args, true));
     } else {
       TRI_ASSERT(false);
     }

diff --git a/arangod/Aql/OptimizerRulesReplaceFunctions.cpp b/arangod/Aql/OptimizerRulesReplaceFunctions.cpp
@@ -282,7 +282,7 @@ AstNode* replaceNearOrWithin(AstNode* funAstNode, ExecutionNode* calcNode,
   argsArray->addMember(accessNodeLon);
   argsArray->addMember(params.latitude);
   argsArray->addMember(params.longitude);
-  auto* funDist = ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("DISTANCE"), argsArray);
+  auto* funDist = ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("DISTANCE"), argsArray, true);
 
   AstNode* expressionAst = funDist;
 
@@ -351,7 +351,7 @@ AstNode* replaceNearOrWithin(AstNode* funAstNode, ExecutionNode* calcNode,
     argsArrayMerge->addMember(obj);
 
     auto* funMerge =
-        ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("MERGE"), argsArrayMerge);
+        ast->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("MERGE"), argsArrayMerge, true);
 
     Variable* calcMergeOutVariable = ast->variables()->createTemporaryVariable();
     auto calcMergeExpr = std::make_unique<Expression>(ast, funMerge);
@@ -465,7 +465,7 @@ AstNode* replaceWithinRectangle(AstNode* funAstNode, ExecutionNode* calcNode,
       fargs->addMember(arr);
     }
   }
-  AstNode* fcall = ast->createNodeFunctionCall("GEO_CONTAINS", fargs);
+  AstNode* fcall = ast->createNodeFunctionCall("GEO_CONTAINS", fargs, true);
 
   // FILTER part
   AstNode* filterNode = ast->createNodeFilter(fcall);

diff --git a/arangod/Aql/grammar.cpp b/arangod/Aql/grammar.cpp
@@ -3861,7 +3861,7 @@ YYLTYPE yylloc = yyloc_default;
 #line 1369 "Aql/grammar.y"
                                                               {
       auto list = static_cast<AstNode const*>(parser->popStack());
-      (yyval.node) = parser->ast()->createNodeFunctionCall(static_cast<char const*>(parser->popStack()), list);
+      (yyval.node) = parser->ast()->createNodeFunctionCall(static_cast<char const*>(parser->popStack()), list, false);
     }
 #line 3867 "Aql/grammar.cpp"
     break;
@@ -3879,7 +3879,7 @@ YYLTYPE yylloc = yyloc_default;
 #line 1376 "Aql/grammar.y"
                                                               {
       auto list = static_cast<AstNode const*>(parser->popStack());
-      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), list);
+      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), list, false);
     }
 #line 3885 "Aql/grammar.cpp"
     break;
@@ -4034,7 +4034,7 @@ YYLTYPE yylloc = yyloc_default;
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember((yyvsp[-3].node));
       arguments->addMember((yyvsp[0].node));
-      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments);
+      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments, false);
       (yyval.node) = parser->ast()->createNodeUnaryOperator(NODE_TYPE_OPERATOR_UNARY_NOT, expression);
     }
 #line 4041 "Aql/grammar.cpp"
@@ -4046,7 +4046,7 @@ YYLTYPE yylloc = yyloc_default;
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember((yyvsp[-3].node));
       arguments->addMember((yyvsp[0].node));
-      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
       (yyval.node) = parser->ast()->createNodeUnaryOperator(NODE_TYPE_OPERATOR_UNARY_NOT, expression);
     }
 #line 4053 "Aql/grammar.cpp"
@@ -4058,7 +4058,7 @@ YYLTYPE yylloc = yyloc_default;
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember((yyvsp[-3].node));
       arguments->addMember((yyvsp[0].node));
-      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
     }
 #line 4064 "Aql/grammar.cpp"
     break;
@@ -4069,7 +4069,7 @@ YYLTYPE yylloc = yyloc_default;
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember((yyvsp[-2].node));
       arguments->addMember((yyvsp[0].node));
-      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments);
+      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments, false);
     }
 #line 4075 "Aql/grammar.cpp"
     break;
@@ -4080,7 +4080,7 @@ YYLTYPE yylloc = yyloc_default;
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember((yyvsp[-2].node));
       arguments->addMember((yyvsp[0].node));
-      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      (yyval.node) = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
     }
 #line 4086 "Aql/grammar.cpp"
     break;
@@ -4091,7 +4091,7 @@ YYLTYPE yylloc = yyloc_default;
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember((yyvsp[-2].node));
       arguments->addMember((yyvsp[0].node));
-      AstNode* node = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      AstNode* node = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
       (yyval.node) = parser->ast()->createNodeUnaryOperator(NODE_TYPE_OPERATOR_UNARY_NOT, node);
     }
 #line 4098 "Aql/grammar.cpp"

diff --git a/arangod/Aql/grammar.y b/arangod/Aql/grammar.y
@@ -1368,14 +1368,14 @@ function_call:
       parser->pushStack(node);
     } optional_function_call_arguments T_CLOSE %prec FUNCCALL {
       auto list = static_cast<AstNode const*>(parser->popStack());
-      $$ = parser->ast()->createNodeFunctionCall(static_cast<char const*>(parser->popStack()), list);
+      $$ = parser->ast()->createNodeFunctionCall(static_cast<char const*>(parser->popStack()), list, false);
     }
   | T_LIKE T_OPEN {
       auto node = parser->ast()->createNodeArray();
       parser->pushStack(node);
     } optional_function_call_arguments T_CLOSE %prec FUNCCALL {
       auto list = static_cast<AstNode const*>(parser->popStack());
-      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), list);
+      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), list, false);
     }
   ;
 
@@ -1441,39 +1441,39 @@ operator_binary:
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember($1);
       arguments->addMember($4);
-      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments);
+      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments, false);
       $$ = parser->ast()->createNodeUnaryOperator(NODE_TYPE_OPERATOR_UNARY_NOT, expression);
     }
   | expression T_NOT T_REGEX_MATCH expression {
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember($1);
       arguments->addMember($4);
-      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      AstNode* expression = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
       $$ = parser->ast()->createNodeUnaryOperator(NODE_TYPE_OPERATOR_UNARY_NOT, expression);
     }
   | expression T_NOT T_REGEX_NON_MATCH expression {
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember($1);
       arguments->addMember($4);
-      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
     }
   | expression T_LIKE expression {
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember($1);
       arguments->addMember($3);
-      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments);
+      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("LIKE"), arguments, false);
     }
   | expression T_REGEX_MATCH expression {
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember($1);
       arguments->addMember($3);
-      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      $$ = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
     }
   | expression T_REGEX_NON_MATCH expression {
       AstNode* arguments = parser->ast()->createNodeArray(2);
       arguments->addMember($1);
       arguments->addMember($3);
-      AstNode* node = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments);
+      AstNode* node = parser->ast()->createNodeFunctionCall(TRI_CHAR_LENGTH_PAIR("REGEX_TEST"), arguments, false);
       $$ = parser->ast()->createNodeUnaryOperator(NODE_TYPE_OPERATOR_UNARY_NOT, node);
     }
   | expression quantifier T_EQ expression {