OGR: Add upsert operation support (OSGeo#6199)

This PR expands the current set of OGR supported feature operations (Create, Set, and Delete, ie. INSERT, UPDATE, and DELETE in SQL parlance) to include UPSERT (ie. if record does not exist then INSERT, else UPDATE). Support for the new UPSERT method has been implemented in the MongoDBv3 and Memory drivers.
OSGeo-jp · Aug 31, 2022 · 9bcd7b4 · 9bcd7b4
1 parent 32f32a6
commit 9bcd7b4
Show file tree

Hide file tree

Showing 29 changed files with 576 additions and 24 deletions.
diff --git a/autotest/ogr/ogr_mem.py b/autotest/ogr/ogr_mem.py
@@ -935,3 +935,40 @@ def test_ogr_mem_arrow_stream_pyarrow():
     assert len(batches) == 1
     arrays = batches[0].flatten()
     assert len(arrays) == 2
+
+
+###############################################################################
+# Test upserting a feature.
+
+
+def test_ogr_mem_upsert_feature():
+    # Create a memory layer
+    lyr = gdaltest.mem_ds.CreateLayer("upsert_feature")
+
+    # Add a string field
+    lyr.CreateField(ogr.FieldDefn("test", ogr.OFTString))
+
+    # Create a feature with some data
+    f = ogr.Feature(lyr.GetLayerDefn())
+    f.SetField("test", "original")
+    assert lyr.CreateFeature(f) == 0
+    assert f.GetFID() == 0
+
+    # Upsert an existing feature
+    f = ogr.Feature(lyr.GetLayerDefn())
+    f.SetFID(0)
+    f.SetField("test", "updated")
+    assert lyr.UpsertFeature(f) == 0
+
+    # Verify that we have set an existing feature
+    f = lyr.GetFeature(0)
+    assert f is not None
+    assert f.GetField("test") == "updated"
+
+    # Upsert a new feature
+    f = ogr.Feature(lyr.GetLayerDefn())
+    f.SetFID(1)
+    assert lyr.UpsertFeature(f) == 0
+
+    # Verify that we have created a feature
+    assert lyr.GetFeature(1) is not None
diff --git a/autotest/ogr/ogr_mongodbv3.py b/autotest/ogr/ogr_mongodbv3.py
@@ -103,6 +103,7 @@ def test_ogr_mongodbv3_init():
     ogrtest.mongodbv3_layer_name_guess_types = None
     ogrtest.mongodbv3_layer_name_with_2d_index = None
     ogrtest.mongodbv3_layer_name_no_spatial_index = None
+    ogrtest.mongodbv3_layer_name_upsert_feature = None
 
     ds = ogr.Open(ogrtest.mongodbv3_test_uri)
     if ds is None:
@@ -853,6 +854,12 @@ def test_ogr_mongodbv3_2():
     gdal.PopErrorHandler()
     assert ret != 0
 
+    # Upsert fails in read-only
+    gdal.PushErrorHandler()
+    ret = lyr.UpsertFeature(f)
+    gdal.PopErrorHandler()
+    assert ret != 0
+
 
 ###############################################################################
 # test_ogrsf
@@ -873,6 +880,58 @@ def test_ogr_mongodbv3_3():
     assert ret.find("INFO") != -1 and ret.find("ERROR") == -1
 
 
+###############################################################################
+# Test upserting a feature.
+
+
+def test_ogr_mongodbv3_upsert_feature():
+    if ogrtest.mongodbv3_drv is None:
+        pytest.skip()
+
+    # Open database in read-write
+    ogrtest.mongodbv3_ds = None
+    ogrtest.mongodbv3_ds = ogr.Open(ogrtest.mongodbv3_test_uri, update=1)
+
+    # Create a layer
+    ogrtest.mongodbv3_layer_name_upsert_feature = (
+        ogrtest.mongodbv3_layer_name + "_upsert_feature"
+    )
+    lyr = ogrtest.mongodbv3_ds.CreateLayer(
+        ogrtest.mongodbv3_layer_name_upsert_feature,
+        geom_type=ogr.wkbNone,
+        options=["FID=", "WRITE_OGR_METADATA=NO"],
+    )
+
+    # Add a string field
+    lyr.CreateField(ogr.FieldDefn("test", ogr.OFTString))
+
+    # Create a feature with some data
+    f = ogr.Feature(lyr.GetLayerDefn())
+    f.SetFID(1)
+    f["_id"] = "000000000000000000000001"  # 24-digit hex MongoDB object id
+    f.SetField("test", "original")
+    assert lyr.CreateFeature(f) == 0
+
+    # Upsert an existing feature
+    f = lyr.GetFeature(1)
+    assert f is not None
+    f.SetField("test", "updated")
+    assert lyr.UpsertFeature(f) == 0
+
+    # Verify that we have set an existing feature
+    f = lyr.GetFeature(1)
+    assert f is not None
+    assert f.GetField("test") == "updated"
+
+    # Upsert a new feature
+    f.SetFID(2)
+    f["_id"] = "000000000000000000000002"
+    assert lyr.UpsertFeature(f) == 0
+
+    # Verify that we have created a feature
+    assert lyr.GetFeatureCount() == 2
+
+
 ###############################################################################
 # Cleanup
 
@@ -904,5 +963,9 @@ def test_ogr_mongodbv3_cleanup():
         ogrtest.mongodbv3_ds.ExecuteSQL(
             "DELLAYER:" + ogrtest.mongodbv3_layer_name_no_spatial_index
         )
+    if ogrtest.mongodbv3_layer_name_upsert_feature is not None:
+        ogrtest.mongodbv3_ds.ExecuteSQL(
+            "DELLAYER:" + ogrtest.mongodbv3_layer_name_upsert_feature
+        )
 
     ogrtest.mongodbv3_ds = None
diff --git a/ogr/ogr_api.h b/ogr/ogr_api.h
@@ -668,6 +668,7 @@ OGRFeatureH CPL_DLL OGR_L_GetFeature( OGRLayerH, GIntBig )  CPL_WARN_UNUSED_RESU
 OGRErr CPL_DLL OGR_L_SetFeature( OGRLayerH, OGRFeatureH ) CPL_WARN_UNUSED_RESULT;
 OGRErr CPL_DLL OGR_L_CreateFeature( OGRLayerH, OGRFeatureH ) CPL_WARN_UNUSED_RESULT;
 OGRErr CPL_DLL OGR_L_DeleteFeature( OGRLayerH, GIntBig ) CPL_WARN_UNUSED_RESULT;
+OGRErr CPL_DLL OGR_L_UpsertFeature( OGRLayerH, OGRFeatureH ) CPL_WARN_UNUSED_RESULT;
 OGRFeatureDefnH CPL_DLL OGR_L_GetLayerDefn( OGRLayerH );
 OGRSpatialReferenceH CPL_DLL OGR_L_GetSpatialRef( OGRLayerH );
 int    CPL_DLL OGR_L_FindFieldIndex( OGRLayerH, const char *, int bExactMatch );

diff --git a/ogr/ogr_core.h b/ogr/ogr_core.h
@@ -881,6 +881,7 @@ int CPL_DLL OGRParseDate( const char *pszInput, OGRField *psOutput,
 #define OLCAlterGeomFieldDefn  "AlterGeomFieldDefn" /**< Layer capability for geometry field alteration */
 #define OLCTransactions        "Transactions"       /**< Layer capability for transactions */
 #define OLCDeleteFeature       "DeleteFeature"      /**< Layer capability for feature deletion */
+#define OLCUpsertFeature       "UpsertFeature"      /**< Layer capability for feature upsert */
 #define OLCFastSetNextByIndex  "FastSetNextByIndex" /**< Layer capability for setting next feature index */
 #define OLCStringsAsUTF8       "StringsAsUTF8"      /**< Layer capability for strings returned with UTF-8 encoding */
 #define OLCIgnoreFields        "IgnoreFields"       /**< Layer capability for field ignoring */

diff --git a/ogr/ograpispy.cpp b/ogr/ograpispy.cpp
@@ -941,6 +941,18 @@ void OGRAPISpy_L_CreateFeature( OGRLayerH hLayer, OGRFeatureH hFeat )
     OGRAPISpyFileClose();
 }
 
+void OGRAPISpy_L_UpsertFeature( OGRLayerH hLayer, OGRFeatureH hFeat )
+{
+    CPLMutexHolderD(&hMutex);
+    OGRAPISpyFlushDefered();
+    OGRAPISpyDumpFeature(hFeat);
+    fprintf(fpSpyFile, "%s.UpsertFeature(f)\n",
+            OGRAPISpyGetLayerVar(hLayer).c_str());
+    // In case layer defn is changed afterwards.
+    fprintf(fpSpyFile, "f = None\n");
+    OGRAPISpyFileClose();
+}
+
 static void OGRAPISpyDumpFieldDefn( OGRFieldDefn* poFieldDefn )
 {
     CPLMutexHolderD(&hMutex);

diff --git a/ogr/ograpispy.h b/ogr/ograpispy.h
@@ -114,6 +114,7 @@ void OGRAPISpy_L_SetNextByIndex( OGRLayerH hLayer, GIntBig nIndex );
 void OGRAPISpy_L_GetNextFeature( OGRLayerH hLayer );
 void OGRAPISpy_L_SetFeature( OGRLayerH hLayer, OGRFeatureH hFeat );
 void OGRAPISpy_L_CreateFeature( OGRLayerH hLayer, OGRFeatureH hFeat );
+void OGRAPISpy_L_UpsertFeature( OGRLayerH hLayer, OGRFeatureH hFeat );
 void OGRAPISpy_L_CreateField( OGRLayerH hLayer, OGRFieldDefnH hField,
                               int bApproxOK );
 void OGRAPISpy_L_DeleteField( OGRLayerH hLayer, int iField );

diff --git a/ogr/ogrsf_frmts/generic/ogreditablelayer.cpp b/ogr/ogrsf_frmts/generic/ogreditablelayer.cpp
@@ -428,6 +428,22 @@ OGRErr      OGREditableLayer::ICreateFeature( OGRFeature *poFeature )
     return eErr;
 }
 
+/************************************************************************/
+/*                          IUpsertFeature()                            */
+/************************************************************************/
+
+OGRErr      OGREditableLayer::IUpsertFeature( OGRFeature* poFeature )
+{
+   if( GetFeature(poFeature->GetFID()) )
+   {
+      return ISetFeature(poFeature);
+   }
+   else
+   {
+      return ICreateFeature(poFeature);
+   }
+}
+
 /************************************************************************/
 /*                          DeleteFeature()                             */
 /************************************************************************/

diff --git a/ogr/ogrsf_frmts/generic/ogreditablelayer.h b/ogr/ogrsf_frmts/generic/ogreditablelayer.h
@@ -100,6 +100,7 @@ class CPL_DLL OGREditableLayer : public OGRLayerDecorator
     virtual OGRFeature *GetFeature( GIntBig nFID ) override;
     virtual OGRErr      ISetFeature( OGRFeature *poFeature ) override;
     virtual OGRErr      ICreateFeature( OGRFeature *poFeature ) override;
+    virtual OGRErr      IUpsertFeature( OGRFeature* poFeature ) override;
     virtual OGRErr      DeleteFeature( GIntBig nFID ) override;
 
     virtual OGRwkbGeometryType GetGeomType() override;

diff --git a/ogr/ogrsf_frmts/generic/ogremulatedtransaction.cpp b/ogr/ogrsf_frmts/generic/ogremulatedtransaction.cpp
@@ -68,6 +68,7 @@ class OGRLayerWithTransaction final: public OGRLayerDecorator
     virtual OGRFeature *GetFeature( GIntBig nFID ) override;
     virtual OGRErr      ISetFeature( OGRFeature *poFeature ) override;
     virtual OGRErr      ICreateFeature( OGRFeature *poFeature ) override;
+    virtual OGRErr      IUpsertFeature( OGRFeature* poFeature ) override;
 
     virtual OGRErr      Rename(const char* pszNewName) override;
 };
@@ -696,6 +697,17 @@ OGRErr       OGRLayerWithTransaction::ICreateFeature( OGRFeature *poFeature )
     return eErr;
 }
 
+OGRErr       OGRLayerWithTransaction::IUpsertFeature( OGRFeature* poFeature )
+{
+    if( !m_poDecoratedLayer ) return OGRERR_FAILURE;
+    OGRFeature* poSrcFeature = new OGRFeature(m_poDecoratedLayer->GetLayerDefn());
+    poSrcFeature->SetFrom(poFeature);
+    poSrcFeature->SetFID(poFeature->GetFID());
+    OGRErr eErr = m_poDecoratedLayer->UpsertFeature(poSrcFeature);
+    delete poSrcFeature;
+    return eErr;
+}
+
 OGRErr OGRLayerWithTransaction::Rename(const char* pszNewName)
 {
     if( !m_poDecoratedLayer ) return OGRERR_FAILURE;

diff --git a/ogr/ogrsf_frmts/generic/ogrlayer.cpp b/ogr/ogrsf_frmts/generic/ogrlayer.cpp
@@ -669,6 +669,44 @@ OGRErr OGR_L_CreateFeature( OGRLayerH hLayer, OGRFeatureH hFeat )
     return OGRLayer::FromHandle(hLayer)->CreateFeature( OGRFeature::FromHandle(hFeat) );
 }
 
+/************************************************************************/
+/*                           UpsertFeature()                           */
+/************************************************************************/
+
+OGRErr OGRLayer::UpsertFeature( OGRFeature *poFeature )
+
+{
+    ConvertGeomsIfNecessary(poFeature);
+    return IUpsertFeature(poFeature);
+}
+
+/************************************************************************/
+/*                           IUpsertFeature()                           */
+/************************************************************************/
+
+OGRErr OGRLayer::IUpsertFeature( OGRFeature * )
+{
+    return OGRERR_UNSUPPORTED_OPERATION;
+}
+
+/************************************************************************/
+/*                        OGR_L_UpsertFeature()                         */
+/************************************************************************/
+
+OGRErr OGR_L_UpsertFeature( OGRLayerH hLayer, OGRFeatureH hFeat )
+
+{
+    VALIDATE_POINTER1( hLayer, "OGR_L_UpsertFeature", OGRERR_INVALID_HANDLE );
+    VALIDATE_POINTER1( hFeat, "OGR_L_UpsertFeature", OGRERR_INVALID_HANDLE );
+
+#ifdef OGRAPISPY_ENABLED
+    if( bOGRAPISpyEnabled )
+        OGRAPISpy_L_UpsertFeature(hLayer, hFeat);
+#endif
+
+    return OGRLayer::FromHandle(hLayer)->UpsertFeature( OGRFeature::FromHandle(hFeat) );
+}
+
 /************************************************************************/
 /*                            CreateField()                             */
 /************************************************************************/

diff --git a/ogr/ogrsf_frmts/generic/ogrlayerdecorator.cpp b/ogr/ogrsf_frmts/generic/ogrlayerdecorator.cpp
@@ -139,6 +139,12 @@ OGRErr      OGRLayerDecorator::ICreateFeature( OGRFeature *poFeature )
     return m_poDecoratedLayer->CreateFeature(poFeature);
 }
 
+OGRErr      OGRLayerDecorator::IUpsertFeature( OGRFeature* poFeature )
+{
+   if ( !m_poDecoratedLayer ) return OGRERR_FAILURE;
+   return m_poDecoratedLayer->UpsertFeature(poFeature);
+}
+
 OGRErr      OGRLayerDecorator::DeleteFeature( GIntBig nFID )
 {
     if( !m_poDecoratedLayer ) return OGRERR_FAILURE;

diff --git a/ogr/ogrsf_frmts/generic/ogrlayerdecorator.h b/ogr/ogrsf_frmts/generic/ogrlayerdecorator.h
@@ -64,6 +64,7 @@ class CPL_DLL OGRLayerDecorator : public OGRLayer
     virtual OGRFeature *GetFeature( GIntBig nFID ) override;
     virtual OGRErr      ISetFeature( OGRFeature *poFeature ) override;
     virtual OGRErr      ICreateFeature( OGRFeature *poFeature ) override;
+    virtual OGRErr      IUpsertFeature( OGRFeature* poFeature ) override;
     virtual OGRErr      DeleteFeature( GIntBig nFID ) override;
 
     virtual GDALDataset* GetDataset() override;

diff --git a/ogr/ogrsf_frmts/generic/ogrlayerpool.cpp b/ogr/ogrsf_frmts/generic/ogrlayerpool.cpp
@@ -353,6 +353,16 @@ OGRErr      OGRProxiedLayer::ICreateFeature( OGRFeature *poFeature )
     return poUnderlyingLayer->CreateFeature(poFeature);
 }
 
+/************************************************************************/
+/*                            IUpsertFeature()                          */
+/************************************************************************/
+
+OGRErr      OGRProxiedLayer::IUpsertFeature( OGRFeature* poFeature )
+{
+   if ( poUnderlyingLayer == nullptr && !OpenUnderlyingLayer() ) return OGRERR_FAILURE;
+   return poUnderlyingLayer->UpsertFeature(poFeature);
+}
+
 /************************************************************************/
 /*                           DeleteFeature()                            */
 /************************************************************************/

diff --git a/ogr/ogrsf_frmts/generic/ogrlayerpool.h b/ogr/ogrsf_frmts/generic/ogrlayerpool.h
@@ -130,6 +130,7 @@ class OGRProxiedLayer : public OGRAbstractProxiedLayer
     virtual OGRFeature *GetFeature( GIntBig nFID ) override;
     virtual OGRErr      ISetFeature( OGRFeature *poFeature ) override;
     virtual OGRErr      ICreateFeature( OGRFeature *poFeature ) override;
+    virtual OGRErr      IUpsertFeature( OGRFeature* poFeature ) override;
     virtual OGRErr      DeleteFeature( GIntBig nFID ) override;
 
     virtual GDALDataset* GetDataset() override;

diff --git a/ogr/ogrsf_frmts/generic/ogrmutexedlayer.cpp b/ogr/ogrsf_frmts/generic/ogrmutexedlayer.cpp
@@ -130,6 +130,12 @@ OGRErr      OGRMutexedLayer::ICreateFeature( OGRFeature *poFeature )
     return OGRLayerDecorator::ICreateFeature(poFeature);
 }
 
+OGRErr      OGRMutexedLayer::IUpsertFeature( OGRFeature* poFeature )
+{
+   CPLMutexHolderOptionalLockD(m_hMutex);
+   return OGRLayerDecorator::IUpsertFeature(poFeature);
+}
+
 OGRErr      OGRMutexedLayer::DeleteFeature( GIntBig nFID )
 {
     CPLMutexHolderOptionalLockD(m_hMutex);

diff --git a/ogr/ogrsf_frmts/generic/ogrmutexedlayer.h b/ogr/ogrsf_frmts/generic/ogrmutexedlayer.h
@@ -75,6 +75,7 @@ class CPL_DLL OGRMutexedLayer : public OGRLayerDecorator
     virtual OGRFeature *GetFeature( GIntBig nFID ) override;
     virtual OGRErr      ISetFeature( OGRFeature *poFeature ) override;
     virtual OGRErr      ICreateFeature( OGRFeature *poFeature ) override;
+    virtual OGRErr      IUpsertFeature( OGRFeature* poFeature ) override;
     virtual OGRErr      DeleteFeature( GIntBig nFID ) override;
 
     virtual GDALDataset* GetDataset() override;

diff --git a/ogr/ogrsf_frmts/generic/ogrunionlayer.cpp b/ogr/ogrsf_frmts/generic/ogrunionlayer.cpp
@@ -877,6 +877,22 @@ OGRErr OGRUnionLayer::ISetFeature( OGRFeature* poFeature )
     return OGRERR_FAILURE;
 }
 
+/************************************************************************/
+/*                          IUpsertFeature()                            */
+/************************************************************************/
+
+OGRErr OGRUnionLayer::IUpsertFeature( OGRFeature* poFeature )
+{
+   if( GetFeature(poFeature->GetFID()) )
+   {
+      return ISetFeature(poFeature);
+   }
+   else
+   {
+      return ICreateFeature(poFeature);
+   }
+}
+
 /************************************************************************/
 /*                           GetSpatialRef()                            */
 /************************************************************************/

diff --git a/ogr/ogrsf_frmts/generic/ogrunionlayer.h b/ogr/ogrsf_frmts/generic/ogrunionlayer.h
@@ -133,6 +133,8 @@ class CPL_DLL OGRUnionLayer final: public OGRLayer
 
     virtual OGRErr      ISetFeature( OGRFeature* poFeature ) override;
 
+    virtual OGRErr      IUpsertFeature( OGRFeature* poFeature ) override;
+
     virtual OGRFeatureDefn *GetLayerDefn() override;
 
     virtual OGRSpatialReference *GetSpatialRef() override;