add test cases for malformed and missing info files

DangZhonghua · May 16, 2024 · ee1c21e · ee1c21e
1 parent 18d0edb
commit ee1c21e
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 3 deletions.
diff --git a/scripts/run_extension_medata_tests.sh b/scripts/run_extension_medata_tests.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# Generates a bunch of directories to be used for testing extension updating related behaviour used in `test/extension/update_extensions_ci.test`
+
 # Please consider your energy footprint by only running this script with ccache.
 # note that subsequent runs used cached artifacts, use `make clean` or rm -rf build/debug to clean
 
@@ -25,6 +27,9 @@ export LOCAL_EXTENSION_REPO_VERSION_AND_PLATFORM_INCORRECT="$TEST_DIR/repository
 # Directory containing the extensions for direct installing
 export DIRECT_INSTALL_DIR="$TEST_DIR/direct_install"
 
+# Extension dir with a malformed info file for an extension
+export LOCAL_EXTENSION_DIR_MALFORMED_INFO="$TEST_DIR/extension_dir_malformed_info"
+
 if [ -d "$TEST_DIR_COPY" ]; then
   # REUSE PREVIOUSLY GENERATED DATA
   rm -r $TEST_DIR
@@ -46,6 +51,7 @@ else
   duckdb_extension_load(json DONT_LINK EXTENSION_VERSION v0.0.1)
   duckdb_extension_load(tpch DONT_LINK EXTENSION_VERSION v0.0.1)
   duckdb_extension_load(tpcds DONT_LINK EXTENSION_VERSION v0.0.1)
+  duckdb_extension_load(inet DONT_LINK EXTENSION_VERSION v0.0.1)
 EOL
 
   # Build the extensions using the first config
@@ -58,10 +64,16 @@ EOL
   cp $DUCKDB_BUILD_DIR/extension/tpcds/tpcds.duckdb_extension $DIRECT_INSTALL_DIR/tpcds.duckdb_extension
   $DUCKDB_BUILD_DIR/duckdb -unsigned -c "set extension_directory='$LOCAL_EXTENSION_DIR'; install '$DIRECT_INSTALL_DIR/tpcds.duckdb_extension';"
 
-  # Set updated extension config where we update the tpch extension but not the json extension
+  # Delete the info file from the inet extension
+  DUCKDB_VERSION=`$DUCKDB_BUILD_DIR/duckdb -csv -noheader  -c 'select source_id from pragma_version()'`
+  DUCKDB_PLATFORM=`cat $DUCKDB_BUILD_DIR/duckdb_platform_out`
+  rm $LOCAL_EXTENSION_DIR_MISSING_INFO/$DUCKDB_VERSION/$DUCKDB_PLATFORM/inet.duckdb_extension.info
+
+  # Set updated extension config where we update the tpch and inet extension but not the json extension
   cat > $TEST_DIR/extension_config_after.cmake <<EOL
   duckdb_extension_load(json DONT_LINK EXTENSION_VERSION v0.0.1)
   duckdb_extension_load(tpch DONT_LINK EXTENSION_VERSION v0.0.2)
+  duckdb_extension_load(inet DONT_LINK EXTENSION_VERSION v0.0.2)
 EOL
 
   # Build the extensions using the second config
@@ -114,7 +126,7 @@ EOL
   cp $DUCKDB_BUILD_DIR/extension/json/json.duckdb_extension $DIRECT_INSTALL_DIR/json.duckdb_extension
 
   ###########################
-  ### Prepare malformed repos
+  ### Prepare malformed repos/dirs
   ###########################
   # Build clean duckdb
   rm -rf $DUCKDB_BUILD_DIR
@@ -125,6 +137,12 @@ EOL
   $DUCKDB_BUILD_DIR/duckdb -unsigned -c "set allow_extensions_metadata_mismatch=true; set extension_directory='$LOCAL_EXTENSION_REPO_INCORRECT_DUCKDB_VERSION'; install '$DIRECT_INSTALL_DIR/json_incorrect_version.duckdb_extension'"
   $DUCKDB_BUILD_DIR/duckdb -unsigned -c "set allow_extensions_metadata_mismatch=true; set extension_directory='$LOCAL_EXTENSION_REPO_VERSION_AND_PLATFORM_INCORRECT'; install '$DIRECT_INSTALL_DIR/json_incorrect_version_and_platform.duckdb_extension'"
 
+  # Create dir with malformed info file
+  DUCKDB_VERSION=`$DUCKDB_BUILD_DIR/duckdb -csv -noheader  -c 'select source_id from pragma_version()'`
+  DUCKDB_PLATFORM=`cat $DUCKDB_BUILD_DIR/duckdb_platform_out`
+  $DUCKDB_BUILD_DIR/duckdb -unsigned -c "set extension_directory='$LOCAL_EXTENSION_DIR_MALFORMED_INFO'; install '$DIRECT_INSTALL_DIR/tpcds.duckdb_extension';"
+  echo blablablab > $LOCAL_EXTENSION_DIR_MALFORMED_INFO/$DUCKDB_VERSION/$DUCKDB_PLATFORM/tpcds.duckdb_extension.info
+
   ###################################################################
   ### Allow using copy instead of regenerating test data on every run
   ###################################################################

diff --git a/src/storage/serialization/serialize_parse_info.cpp b/src/storage/serialization/serialize_parse_info.cpp
@@ -332,6 +332,7 @@ void LoadInfo::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty<LoadType>(201, "load_type", load_type);
 	serializer.WritePropertyWithDefault<string>(202, "repository", repository);
 	serializer.WritePropertyWithDefault<string>(203, "version", version);
+	serializer.WritePropertyWithDefault<bool>(204, "repo_is_alias", repo_is_alias);
 }
 
 unique_ptr<ParseInfo> LoadInfo::Deserialize(Deserializer &deserializer) {
@@ -340,6 +341,7 @@ unique_ptr<ParseInfo> LoadInfo::Deserialize(Deserializer &deserializer) {
 	deserializer.ReadProperty<LoadType>(201, "load_type", result->load_type);
 	deserializer.ReadPropertyWithDefault<string>(202, "repository", result->repository);
 	deserializer.ReadPropertyWithDefault<string>(203, "version", result->version);
+	deserializer.ReadPropertyWithDefault<bool>(204, "repo_is_alias", result->repo_is_alias);
 	return std::move(result);
 }
 

diff --git a/test/extension/update_extensions_ci.test b/test/extension/update_extensions_ci.test
@@ -28,6 +28,9 @@ require-env LOCAL_EXTENSION_DIR
 # This dir holds some directly installable, incorrectly matched extensions, but also a correctly installable
 require-env DIRECT_INSTALL_DIR
 
+# This extension dir is expected to contain tpch and tpcds, but with a corrupted tpcds metadata file 
+require-env LOCAL_EXTENSION_DIR_MALFORMED_INFO
+
 # Parquet is statically loaded for this test
 require parquet
 
@@ -40,6 +43,7 @@ set extension_directory='${LOCAL_EXTENSION_DIR}'
 query IIII rowsort
 SELECT extension_name, install_mode, installed_from, extension_version FROM duckdb_extensions() where installed and extension_name != 'jemalloc' and extension_name != 'parquet'
 ----
+inet	UNKNOWN	(empty)	(empty)
 json	REPOSITORY	./build/extension_metadata_test_data/repository	v0.0.1
 tpcds	CUSTOM_PATH	./build/extension_metadata_test_data/direct_install/tpcds.duckdb_extension	v0.0.1
 tpch	REPOSITORY	./build/extension_metadata_test_data/repository	v0.0.1
@@ -63,6 +67,7 @@ select source_id from pragma_version();
 query IIIII rowsort
 UPDATE EXTENSIONS;
 ----
+inet	(empty)	MISSING_INSTALL_INFO	(empty)	(empty)
 json	<REGEX>:.*	NO_UPDATE_AVAILABLE	v0.0.1	v0.0.1
 tpcds	<REGEX>:.*	NOT_A_REPOSITORY	v0.0.1	v0.0.1
 tpch	<REGEX>:.*	UPDATED	v0.0.1	v0.0.2
@@ -71,14 +76,20 @@ tpch	<REGEX>:.*	UPDATED	v0.0.1	v0.0.2
 query IIII rowsort
 SELECT extension_name, install_mode, installed_from, extension_version FROM duckdb_extensions() where installed and extension_name != 'jemalloc' and extension_name != 'parquet'
 ----
+inet	UNKNOWN	(empty)	(empty)
 json	REPOSITORY	./build/extension_metadata_test_data/repository	v0.0.1
 tpcds	CUSTOM_PATH	./build/extension_metadata_test_data/direct_install/tpcds.duckdb_extension	v0.0.1
 tpch	REPOSITORY	./build/extension_metadata_test_data/repository	v0.0.2
 
-# Rerunning update will now show everything being up-to-date
+# Now lets restored the corrupt inet extension (it has a missing info file)
+statement ok
+FORCE INSTALL inet;
+
+# Rerunning update will now show everything being up-to-date (inet extension was force installed and is now v0.0.2)
 query IIIII rowsort
 UPDATE EXTENSIONS;
 ----
+inet	<REGEX>:.*	NO_UPDATE_AVAILABLE	v0.0.2	v0.0.2
 json	<REGEX>:.*	NO_UPDATE_AVAILABLE	v0.0.1	v0.0.1
 tpcds	<REGEX>:.*	NOT_A_REPOSITORY	v0.0.1	v0.0.1
 tpch	<REGEX>:.*	NO_UPDATE_AVAILABLE	v0.0.2	v0.0.2
@@ -92,10 +103,14 @@ load tpch;
 statement ok
 load tpcds;
 
+statement ok
+load inet;
+
 # Ensure the result is still fine after loading; this will ensure Version() call matches the encoded footer value
 query IIII rowsort
 SELECT extension_name, install_mode, installed_from, extension_version FROM duckdb_extensions() where installed and extension_name != 'jemalloc' and extension_name != 'parquet'
 ----
+inet	REPOSITORY	./build/extension_metadata_test_data/repository	v0.0.2
 json	REPOSITORY	./build/extension_metadata_test_data/repository	v0.0.1
 tpcds	CUSTOM_PATH	./build/extension_metadata_test_data/direct_install/tpcds.duckdb_extension	v0.0.1
 tpch	REPOSITORY	./build/extension_metadata_test_data/repository	v0.0.2
@@ -185,3 +200,36 @@ statement error
 LOAD json;
 ----
  Also, the file was built for the platform 'test_platform', but we can only load extensions built for platform
+
+restart
+
+statement ok
+set extension_directory='${LOCAL_EXTENSION_DIR_MALFORMED_INFO}'
+
+# this will now throw IOError
+statement error
+FROM duckdb_extensions() where installed and extension_name != 'jemalloc' and extension_name != 'parquet'
+----
+IO Error: Failed to read info file for 'tpcds' extension
+
+# this will now throw IOError
+statement error
+UPDATE EXTENSIONS
+----
+IO Error: Failed to read info file for 'tpcds' extension
+
+# lets restore by reinstalling tpcds
+statement ok
+FORCE INSTALL '${DIRECT_INSTALL_DIR}/tpcds.duckdb_extension';
+
+# Things are back to normal
+query IIII
+SELECT extension_name, install_mode, installed_from, extension_version FROM duckdb_extensions() where installed and extension_name != 'jemalloc' and extension_name != 'parquet'
+----
+tpcds	CUSTOM_PATH	./build/extension_metadata_test_data/direct_install/tpcds.duckdb_extension	v0.0.1
+
+# Same here
+query IIIII
+UPDATE EXTENSIONS
+----
+tpcds	(empty)	NOT_A_REPOSITORY	v0.0.1	v0.0.1