[11999.5] Support loading tables with default filling into SQL-Loader

* Add special field markers to fill a column in SQLStorage with default values
* Add a safeguard to DBCStorage::EraseEntry, Add helper function DBCStorage::InsertEntry to store content

Note: If required similar adjustments to DBCStorage might be reasonable if required

src/shared/Database/DBCFileLoader.h

@@ -24,8 +24,10 @@
 
 enum
 {
-    FT_NA='x',                                              //not used or unknown, 4 byte size
+    FT_NA='x',                                              // ignore/ default, 4 byte size, in Source String means field is ignored, in Dest String means field is filled with default value
-    FT_NA_BYTE='X',                                         //not used or unknown, byte
+    FT_NA_BYTE='X',                                         // ignore/ default, 1 byte size, see above
+    FT_NA_FLOAT='F',                                        // ignore/ default,  float size, see above
+    FT_NA_POINTER='p',                                      // fill default value into dest, pointer size, Use this only with static data (otherwise mem-leak)
     FT_STRING='s',                                          //char*
     FT_FLOAT='f',                                           //float
     FT_INT='i',                                             //uint32

src/shared/Database/DBCStore.h

@@ -88,7 +88,8 @@ class DBCStorage
             nCount = 0;
         }
 
-        void EraseEntry(uint32 id) { indexTable[id] = NULL; }
+        void EraseEntry(uint32 id) { assert(id < nCount && "To be erased entry must be in bounds!") ; indexTable[id] = NULL; }
+        void InsertEntry(T* entry, uint32 id) { assert(id < nCount && "To be inserted entry must be in bounds!"); indexTable[id] = entry; }
 
     private:
         uint32 nCount;

src/shared/Database/SQLStorage.cpp

@@ -22,7 +22,7 @@
 void SQLStorage::EraseEntry(uint32 id)
 {
     uint32 offset = 0;
-    for(uint32 x = 0; x < iNumFields; ++x)
+    for (uint32 x = 0; x < oNumFields; ++x)
     {
         switch(dst_format[x])
         {
@@ -43,8 +43,13 @@ void SQLStorage::EraseEntry(uint32 id)
                 break;
             }
             case FT_NA:
+                offset += sizeof(uint32); break;
             case FT_NA_BYTE:
-                break;
+                offset += sizeof(char);   break;
+            case FT_NA_FLOAT:
+                offset += sizeof(float);  break;
+            case FT_NA_POINTER:
+                offset += sizeof(char*);  break;
             case FT_IND:
             case FT_SORT:
                 assert(false && "SQL storage not have sort field types");
@@ -61,7 +66,7 @@ void SQLStorage::EraseEntry(uint32 id)
 void SQLStorage::Free ()
 {
     uint32 offset = 0;
-    for(uint32 x = 0; x < iNumFields; ++x)
+    for (uint32 x = 0; x < oNumFields; ++x)
     {
         switch(dst_format[x])
         {
@@ -83,8 +88,13 @@ void SQLStorage::Free ()
                 break;
             }
             case FT_NA:
+                offset += sizeof(uint32); break;
             case FT_NA_BYTE:
-                break;
+                offset += sizeof(char);   break;
+            case FT_NA_FLOAT:
+                offset += sizeof(float);  break;
+            case FT_NA_POINTER:
+                offset += sizeof(char*);  break;
             case FT_IND:
             case FT_SORT:
                 assert(false && "SQL storage not have sort field types");

src/shared/Database/SQLStorage.h

@@ -62,6 +62,7 @@ class SQLStorage
         uint32 RecordCount;
         uint32 MaxEntry;
         uint32 iNumFields;
+        uint32 oNumFields;
 
         char const* GetTableName() const { return table; }
 
@@ -77,6 +78,7 @@ class SQLStorage
             data=NULL;
             pIndex=NULL;
             iNumFields = strlen(src_format);
+            oNumFields = strlen(dst_format);
             MaxEntry = 0;
         }
 
@@ -103,6 +105,9 @@ struct SQLStorageLoaderBase
         template<class D>
             void convert_from_str(uint32 field_pos, char const* src, D& dst);
         void convert_str_to_str(uint32 field_pos, char const* src, char *&dst);
+        template<class S, class D>
+            void default_fill(uint32 field_pos, S src, D &dst);
+        void default_fill_to_str(uint32 field_pos, char const* src, char * & dst);
 
         // trap, no body
         template<class D>

src/shared/Database/SQLStorageImpl.h

@@ -61,6 +61,20 @@ void SQLStorageLoaderBase<T>::convert_from_str(uint32 /*field_pos*/, char const*
     dst = 0;
 }
 
+template<class T>
+template<class S, class D>
+void SQLStorageLoaderBase<T>::default_fill(uint32 /*field_pos*/, S src, D &dst)
+{
+    dst = D(src);
+}
+
+template<class T>
+void SQLStorageLoaderBase<T>::default_fill_to_str(uint32 /*field_pos*/, char const* /*src*/, char * & dst)
+{
+    dst = new char[1];
+    *dst = 0;
+}
+
 template<class T>
 template<class V>
 void SQLStorageLoaderBase<T>::storeValue(V value, SQLStorage &store, char *p, uint32 x, uint32 &offset)
@@ -89,7 +103,16 @@ void SQLStorageLoaderBase<T>::storeValue(V value, SQLStorage &store, char *p, ui
             offset+=sizeof(char*);
             break;
         case FT_NA:
+            subclass->default_fill(x, value, *((int32*)(&p[offset])) );
+            offset+=sizeof(uint32);
+            break;
         case FT_NA_BYTE:
+            subclass->default_fill(x, value, *((char*)(&p[offset])) );
+            offset+=sizeof(char);
+            break;
+        case FT_NA_FLOAT:
+            subclass->default_fill(x, value, *((float*)(&p[offset])) );
+            offset+=sizeof(float);
             break;
         case FT_IND:
         case FT_SORT:
@@ -127,8 +150,9 @@ void SQLStorageLoaderBase<T>::storeValue(char const* value, SQLStorage &store, c
             subclass->convert_str_to_str(x, value, *((char**)(&p[offset])) );
             offset += sizeof(char*);
             break;
-        case FT_NA:
+        case FT_NA_POINTER:
-        case FT_NA_BYTE:
+            subclass->default_fill_to_str(x, value, *((char**)(&p[offset])) );
+            offset += sizeof(char*);
             break;
         case FT_IND:
         case FT_SORT:
@@ -192,7 +216,7 @@ void SQLStorageLoaderBase<T>::Load(SQLStorage &store, bool error_at_empty /*= tr
     }
 
     // get struct size
-    for(uint32 x = 0; x < store.iNumFields; ++x)
+    for (uint32 x = 0; x < store.oNumFields; ++x)
     {
         switch(store.dst_format[x])
         {
@@ -207,8 +231,13 @@ void SQLStorageLoaderBase<T>::Load(SQLStorage &store, bool error_at_empty /*= tr
             case FT_STRING:
                 recordsize += sizeof(char*);  break;
             case FT_NA:
+                recordsize += sizeof(uint32); break;
             case FT_NA_BYTE:
-                break;
+                recordsize += sizeof(char);   break;
+            case FT_NA_FLOAT:
+                recordsize += sizeof(float);  break;
+            case FT_NA_POINTER:
+                recordsize += sizeof(char*);  break;
             case FT_IND:
             case FT_SORT:
                 assert(false && "SQL storage not have sort field types");
@@ -233,31 +262,52 @@ void SQLStorageLoaderBase<T>::Load(SQLStorage &store, bool error_at_empty /*= tr
         newIndex[fields[0].GetUInt32()] = p;
 
         offset = 0;
-        for(uint32 x = 0; x < store.iNumFields; x++)
+        // dependend on dest-size
-            switch(store.src_format[x])
+        // iterate two indexes: x over dest, y over source, y++ IFF x1=x/X
+        for (uint32 x = 0, y = 0; x < store.oNumFields; ++x)
+        {
+            switch (store.dst_format[x])
+            {
+                // For default fill continue and do not increase y
+                case FT_NA:         storeValue((uint32)0, store, p, x, offset);         continue;
+                case FT_NA_BYTE:    storeValue((char)0, store, p, x, offset);           continue;
+                case FT_NA_FLOAT:   storeValue((float)0.0f, store, p, x, offset);       continue;
+                case FT_NA_POINTER: storeValue((char const*)NULL, store, p, x, offset); continue;
+            }
+
+            // It is required that the input has at least as many columns set as the output requires
+            if (y >= store.iNumFields)
+                assert(false && "SQL storage has too few columns!");
+
+            switch(store.src_format[y])
             {
                 case FT_LOGIC:
-                    storeValue((bool)(fields[x].GetUInt32() > 0), store, p, x, offset); break;
+                    storeValue((bool)(fields[y].GetUInt32() > 0), store, p, x, offset); break;
                 case FT_BYTE:
-                    storeValue((char)fields[x].GetUInt8(), store, p, x, offset); break;
+                    storeValue((char)fields[y].GetUInt8(), store, p, x, offset); break;
                 case FT_INT:
-                    storeValue((uint32)fields[x].GetUInt32(), store, p, x, offset); break;
+                    storeValue((uint32)fields[y].GetUInt32(), store, p, x, offset); break;
                 case FT_FLOAT:
-                    storeValue((float)fields[x].GetFloat(), store, p, x, offset); break;
+                    storeValue((float)fields[y].GetFloat(), store, p, x, offset); break;
                 case FT_STRING:
-                    storeValue((char const*)fields[x].GetString(), store, p, x, offset); break;
+                    storeValue((char const*)fields[y].GetString(), store, p, x, offset); break;
                 case FT_NA:
                 case FT_NA_BYTE:
+                case FT_NA_FLOAT:
                     break;
                 case FT_IND:
                 case FT_SORT:
-                    assert(false && "SQL storage not have sort field types");
+                case FT_NA_POINTER:
+                    assert(false && "SQL storage not have sort or pointer field types");
                     break;
                 default:
                     assert(false && "unknown format character");
             }
+            ++y;
+        }
         ++count;
-    }while( result->NextRow() );
+    }
+    while (result->NextRow());
 
     delete result;