Skip to content

Commit

Permalink
Add cdm-zarr to the TDS (Unidata#445)
Browse files Browse the repository at this point in the history
* Add zarr to tds and tds-platform

* Add zarr test catalog and tests
  • Loading branch information
tdrwenski authored Jan 12, 2024
1 parent 511bfc5 commit e779b59
Show file tree
Hide file tree
Showing 386 changed files with 225 additions and 0 deletions.
1 change: 1 addition & 0 deletions tds-platform/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ dependencies {
api 'edu.ucar:cdm-misc'
api 'edu.ucar:cdm-image'
api 'edu.ucar:cdm-s3'
api 'edu.ucar:cdm-zarr'
api 'edu.ucar:netcdf4'
api 'edu.ucar:cdm-mcidas'
api 'edu.ucar:waterml'
Expand Down
1 change: 1 addition & 0 deletions tds/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ dependencies {
compile 'edu.ucar:cdm-misc'
compile 'edu.ucar:cdm-image'
compile 'edu.ucar:cdm-s3'
compile 'edu.ucar:cdm-zarr'
compile 'edu.ucar:grib'
compile 'edu.ucar:netcdf4'
compile 'edu.ucar:httpservices'
Expand Down
39 changes: 39 additions & 0 deletions tds/src/integrationTests/java/thredds/tds/TestZarr.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package thredds.tds;

import static com.google.common.truth.Truth.assertThat;

import java.nio.charset.StandardCharsets;
import javax.servlet.http.HttpServletResponse;
import org.junit.Ignore;
import org.junit.Test;
import thredds.test.util.TestOnLocalServer;

public class TestZarr {
final private static String ZARR_DIR_PATH = "localContent/zarr/zarr_test_data.zarr";
final private static String ZARR_ZIP_PATH = "localContent/zarr/zarr_test_data.zip";
final private static String ZARR_S3_PATH = "s3-zarr/zarr_test_data.zarr";

@Test
public void shouldOpenZarrDirectory() {
checkWithOpendap(ZARR_DIR_PATH);
}

@Test
public void shouldOpenZarrZip() {
checkWithOpendap(ZARR_ZIP_PATH);
}

@Ignore("Still working on S3 Zarr")
@Test
public void shouldOpenObjectStoreZarrFile() {
checkWithOpendap(ZARR_S3_PATH);
}

private static void checkWithOpendap(String path) {
final String endpoint = TestOnLocalServer.withHttpPath("dodsC/" + path + ".dds");
final byte[] content = TestOnLocalServer.getContent(endpoint, HttpServletResponse.SC_OK);
final String stringContent = new String(content, StandardCharsets.UTF_8);

assertThat(stringContent).contains("Int32 /group_with_attrs/F_order_array[dim0 = 20][dim1 = 20];");
}
}
1 change: 1 addition & 0 deletions tds/src/test/content/thredds/catalog.xml
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@
<catalogRef xlink:title="ESGF" xlink:href="testEsgfProblems.xml" name=""/>
<catalogRef xlink:title="RDA" xlink:href="testRdaProblems.xml" name=""/>
<catalogRef xlink:title="TDS S3" xlink:href="tds-s3.xml" name=""/>
<catalogRef xlink:title="TDS Zarr" xlink:href="tds-zarr.xml" name=""/>
<catalogRef xlink:title="Test nested aggregation" xlink:href="catalogNestedAgg.xml" name=""/>

<catalogScan location="catalogs5" path="catalogs5" name="catalog scan with version 5 features"/>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"zarr_format": 2
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"group_attr": "foo"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"zarr_format": 2
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"chunks": [
4,
5
],
"compressor": null,
"dtype": "<i4",
"fill_value": 0,
"filters": null,
"order": "F",
"shape": [
20,
20
],
"zarr_format": 2
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"bar": "apples",
"baz": [
1,
2,
3,
4
],
"foo": 42
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"chunks": [
10,
10
],
"compressor": null,
"dtype": "<i2",
"fill_value": 0,
"filters": null,
"order": "C",
"shape": [
20,
20
],
"zarr_format": 2,
"dimension_separator": "/"
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"chunks": [
10,
10
],
"compressor": null,
"dtype": "<f4",
"fill_value": 999.0,
"filters": null,
"order": "C",
"shape": [
20,
20
],
"zarr_format": 2
}
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"chunks": [
10,
10
],
"compressor": null,
"dtype": "<f4",
"fill_value": 999.0,
"filters": null,
"order": "C",
"shape": [
20,
20
],
"zarr_format": 2
}
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"chunks": [
10,
10
],
"compressor": null,
"dtype": "<f4",
"fill_value": 999.0,
"filters": null,
"order": "C",
"shape": [
20,
20
],
"zarr_format": 2
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"zarr_format": 2
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"chunks": [
5
],
"compressor": null,
"dtype": "<i4",
"fill_value": 0,
"filters": null,
"order": "C",
"shape": [
20
],
"zarr_format": 2
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"chunks": [
5,
5
],
"compressor": null,
"dtype": "<i4",
"fill_value": 0,
"filters": null,
"order": "C",
"shape": [
20,
20
],
"zarr_format": 2
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"chunks": [
5,
5,
5
],
"compressor": null,
"dtype": "<i4",
"fill_value": 0,
"filters": null,
"order": "C",
"shape": [
20,
20,
20
],
"zarr_format": 2
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"chunks": [
5,
5,
5,
5
],
"compressor": null,
"dtype": "<i4",
"fill_value": 0,
"filters": null,
"order": "C",
"shape": [
20,
20,
20,
20
],
"zarr_format": 2
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading

0 comments on commit e779b59

Please sign in to comment.