Simplify type system: only core types and AttributeTypes

Major simplification of the type system to two categories:
1. Core DataJoint types (no brackets): float32, uuid, bool, json, blob, etc.
2. AttributeTypes (angle brackets): <djblob>, <object>, <attach>, etc.

Changes:
- declare.py: Remove EXTERNAL_TYPES, BINARY_TYPES; simplify to CORE_TYPE_ALIASES + ADAPTED
- heading.py: Remove is_attachment, is_filepath, is_object, is_external flags
- fetch.py: Simplify _get() to only handle uuid, json, blob, and adapters
- table.py: Simplify __make_placeholder() to only handle uuid, json, blob, numeric
- preview.py: Remove special object field handling (will be AttributeType)
- staged_insert.py: Update object type check to use adapter

All special handling (attach, filepath, object, external storage) will be
implemented as built-in AttributeTypes in subsequent phases.

Co-authored-by: dimitri-yatsenko <dimitri@datajoint.com>

Author: claude

src/datajoint/declare.py

@@ -11,13 +11,13 @@
 
 from .attribute_adapter import get_adapter
 from .condition import translate_attribute
-from .errors import FILEPATH_FEATURE_SWITCH, DataJointError, _support_filepath_types
+from .errors import DataJointError
 from .settings import config
 
-UUID_DATA_TYPE = "binary(16)"
-
-# Type aliases for numeric types
-SQL_TYPE_ALIASES = {
+# Core DataJoint type aliases - scientist-friendly names mapped to native SQL types
+# These types can be used without angle brackets in table definitions
+CORE_TYPE_ALIASES = {
+    # Numeric types
     "FLOAT32": "float",
     "FLOAT64": "double",
     "INT64": "bigint",
@@ -29,18 +29,22 @@
     "INT8": "tinyint",
     "UINT8": "tinyint unsigned",
     "BOOL": "tinyint",
+    # UUID type
+    "UUID": "binary(16)",
 }
+
 MAX_TABLE_NAME_LENGTH = 64
 CONSTANT_LITERALS = {
     "CURRENT_TIMESTAMP",
     "NULL",
 }  # SQL literals to be used without quotes (case insensitive)
-EXTERNAL_TABLE_ROOT = "~external"
 
+# Type patterns for declaration parsing
+# Two categories: core type aliases and native passthrough types
 TYPE_PATTERN = {
     k: re.compile(v, re.I)
     for k, v in dict(
-        # Type aliases must come before INTEGER and FLOAT patterns to avoid prefix matching
+        # Core DataJoint type aliases (scientist-friendly names)
         FLOAT32=r"float32$",
         FLOAT64=r"float64$",
         INT64=r"int64$",
@@ -51,51 +55,29 @@
         UINT16=r"uint16$",
         INT8=r"int8$",
         UINT8=r"uint8$",
-        BOOL=r"bool$",  # aliased to tinyint
-        # Native MySQL types
+        BOOL=r"bool$",
+        UUID=r"uuid$",
+        # Native SQL types (passthrough)
         INTEGER=r"((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?|serial$",
         DECIMAL=r"(decimal|numeric)(\s*\(.+\))?(\s+unsigned)?$",
         FLOAT=r"(double|float|real)(\s*\(.+\))?(\s+unsigned)?$",
         STRING=r"(var)?char\s*\(.+\)$",
         JSON=r"json$",
         ENUM=r"enum\s*\(.+\)$",
         TEMPORAL=r"(date|datetime|time|timestamp|year)(\s*\(.+\))?$",
-        INTERNAL_BLOB=r"(tiny|small|medium|long|)blob$",
-        EXTERNAL_BLOB=r"blob@(?P<store>[a-z][\-\w]*)$",
-        INTERNAL_ATTACH=r"attach$",
-        EXTERNAL_ATTACH=r"attach@(?P<store>[a-z][\-\w]*)$",
-        FILEPATH=r"filepath@(?P<store>[a-z][\-\w]*)$",
-        OBJECT=r"object(@(?P<store>[a-z][\-\w]*))?$",  # managed object storage (files/folders)
-        UUID=r"uuid$",
+        BLOB=r"(tiny|small|medium|long|)blob$",
+        # AttributeTypes use angle brackets
         ADAPTED=r"<.+>$",
     ).items()
 }
 
-# custom types are stored in attribute comment
-SPECIAL_TYPES = {
-    "UUID",
-    "INTERNAL_ATTACH",
-    "EXTERNAL_ATTACH",
-    "EXTERNAL_BLOB",
-    "FILEPATH",
-    "OBJECT",
-    "ADAPTED",
-} | set(SQL_TYPE_ALIASES)
+# Types that require special handling (stored in attribute comment for reconstruction)
+SPECIAL_TYPES = {"ADAPTED"} | set(CORE_TYPE_ALIASES)
+
+# Native SQL types that pass through without modification
 NATIVE_TYPES = set(TYPE_PATTERN) - SPECIAL_TYPES
-EXTERNAL_TYPES = {
-    "EXTERNAL_ATTACH",
-    "EXTERNAL_BLOB",
-    "FILEPATH",
-}  # data referenced by a UUID in external tables
-# Blob and attachment types cannot have SQL default values (other than NULL)
-BINARY_TYPES = {
-    "EXTERNAL_ATTACH",
-    "INTERNAL_ATTACH",
-    "EXTERNAL_BLOB",
-    "INTERNAL_BLOB",
-}
 
-assert set().union(SPECIAL_TYPES, EXTERNAL_TYPES, BINARY_TYPES) <= set(TYPE_PATTERN)
+assert SPECIAL_TYPES <= set(TYPE_PATTERN)
 
 
 def match_type(attribute_type):
@@ -459,50 +441,32 @@ def format_attribute(attr):
 
 def substitute_special_type(match, category, foreign_key_sql, context):
     """
+    Substitute special types with their native SQL equivalents.
+
+    Special types are:
+    - Core type aliases (float32 → float, uuid → binary(16), etc.)
+    - ADAPTED types (AttributeTypes in angle brackets)
+
     :param match: dict containing with keys "type" and "comment" -- will be modified in place
     :param category: attribute type category from TYPE_PATTERN
     :param foreign_key_sql: list of foreign key declarations to add to
     :param context: context for looking up user-defined attribute_type adapters
     """
-    if category == "UUID":
-        match["type"] = UUID_DATA_TYPE
-    elif category == "INTERNAL_ATTACH":
-        match["type"] = "LONGBLOB"
-    elif category == "OBJECT":
-        # Object type stores metadata as JSON - no foreign key to external table
-        # Extract store name if present (object@store_name syntax)
-        if "@" in match["type"]:
-            match["store"] = match["type"].split("@", 1)[1]
-        match["type"] = "JSON"
-    elif category in EXTERNAL_TYPES:
-        if category == "FILEPATH" and not _support_filepath_types():
-            raise DataJointError(
-                """
-            The filepath data type is disabled until complete validation.
-            To turn it on as experimental feature, set the environment variable
-            {env} = TRUE or upgrade datajoint.
-            """.format(env=FILEPATH_FEATURE_SWITCH)
-            )
-        match["store"] = match["type"].split("@", 1)[1]
-        match["type"] = UUID_DATA_TYPE
-        foreign_key_sql.append(
-            "FOREIGN KEY (`{name}`) REFERENCES `{{database}}`.`{external_table_root}_{store}` (`hash`) "
-            "ON UPDATE RESTRICT ON DELETE RESTRICT".format(external_table_root=EXTERNAL_TABLE_ROOT, **match)
-        )
-    elif category == "ADAPTED":
+    if category == "ADAPTED":
+        # AttributeType - resolve to underlying dtype
         attr_type, store_name = get_adapter(context, match["type"])
-        # Store the store parameter if present
         if store_name is not None:
             match["store"] = store_name
         match["type"] = attr_type.dtype
+        # Recursively resolve if dtype is also a special type
         category = match_type(match["type"])
         if category in SPECIAL_TYPES:
-            # recursive redefinition from user-defined datatypes.
             substitute_special_type(match, category, foreign_key_sql, context)
-    elif category in SQL_TYPE_ALIASES:
-        match["type"] = SQL_TYPE_ALIASES[category]
+    elif category in CORE_TYPE_ALIASES:
+        # Core type alias - substitute with native SQL type
+        match["type"] = CORE_TYPE_ALIASES[category]
     else:
-        assert False, "Unknown special type"
+        assert False, f"Unknown special type: {category}"
 
 
 def compile_attribute(line, in_key, foreign_key_sql, context):
@@ -513,7 +477,7 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
     :param in_key: set to True if attribute is in primary key set
     :param foreign_key_sql: the list of foreign key declarations to add to
     :param context: context in which to look up user-defined attribute type adapterss
-    :returns: (name, sql, is_external) -- attribute name and sql code for its declaration
+    :returns: (name, sql, store) -- attribute name, sql code for its declaration, and optional store name
     """
     try:
         match = attribute_parser.parseString(line + "#", parseAll=True)
@@ -550,13 +514,10 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
         match["comment"] = ":{type}:{comment}".format(**match)  # insert custom type into comment
         substitute_special_type(match, category, foreign_key_sql, context)
 
-    if category in BINARY_TYPES and match["default"] not in {
-        "DEFAULT NULL",
-        "NOT NULL",
-    }:
-        raise DataJointError(
-            "The default value for blob or attachment attributes can only be NULL in:\n{line}".format(line=line)
-        )
+    # Check for invalid default values on blob types (after type substitution)
+    final_category = match_type(match["type"])
+    if final_category == "BLOB" and match["default"] not in {"DEFAULT NULL", "NOT NULL"}:
+        raise DataJointError("The default value for blob attributes can only be NULL in:\n{line}".format(line=line))
 
     sql = ("`{name}` {type} {default}" + (' COMMENT "{comment}"' if match["comment"] else "")).format(**match)
     return match["name"], sql, match.get("store")

src/datajoint/fetch.py

@@ -1,21 +1,15 @@
-import itertools
 import json
 import numbers
-import uuid
+import uuid as uuid_module
 from functools import partial
-from pathlib import Path
 
 import numpy as np
 import pandas
 
 from datajoint.condition import Top
 
-from . import hash
 from .errors import DataJointError
-from .objectref import ObjectRef
 from .settings import config
-from .storage import StorageBackend
-from .utils import safe_write
 
 
 class key:
@@ -39,79 +33,51 @@ def to_dicts(recarray):
 
 def _get(connection, attr, data, squeeze, download_path):
     """
-    This function is called for every attribute
+    Retrieve and decode attribute data from the database.
+
+    In the simplified type system:
+    - Native types pass through unchanged
+    - JSON types are parsed
+    - UUID types are converted from bytes
+    - Blob types return raw bytes (unless an adapter handles them)
+    - Adapters (AttributeTypes) handle all custom encoding/decoding
 
     :param connection: a dj.Connection object
-    :param attr: attribute name from the table's heading
-    :param data: literal value fetched from the table
-    :param squeeze: if True squeeze blobs
-    :param download_path: for fetches that download data, e.g. attachments
-    :return: unpacked data
+    :param attr: attribute from the table's heading
+    :param data: raw value fetched from the database
+    :param squeeze: if True squeeze blobs (legacy, unused)
+    :param download_path: for fetches that download data (legacy, unused in simplified model)
+    :return: decoded data
     """
     if data is None:
-        return
-    if attr.is_object:
-        # Object type - return ObjectRef handle
-        json_data = json.loads(data) if isinstance(data, str) else data
-        # Get the correct backend based on store name in metadata
-        store_name = json_data.get("store")  # None for default store
-        try:
-            spec = config.get_object_store_spec(store_name)
-            backend = StorageBackend(spec)
-        except DataJointError:
-            backend = None
-        return ObjectRef.from_json(json_data, backend=backend)
+        return None
+
+    # JSON type - parse and optionally decode via adapter
     if attr.json:
-        return json.loads(data)
-
-    extern = connection.schemas[attr.database].external[attr.store] if attr.is_external else None
-
-    # apply custom attribute type decoder if present
-    def adapt(x):
-        return attr.adapter.decode(x, key=None) if attr.adapter else x
-
-    if attr.is_filepath:
-        return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0])
-    if attr.is_attachment:
-        # Steps:
-        # 1. get the attachment filename
-        # 2. check if the file already exists at download_path, verify checksum
-        # 3. if exists and checksum passes then return the local filepath
-        # 4. Otherwise, download the remote file and return the new filepath
-        _uuid = uuid.UUID(bytes=data) if attr.is_external else None
-        attachment_name = extern.get_attachment_name(_uuid) if attr.is_external else data.split(b"\0", 1)[0].decode()
-        local_filepath = Path(download_path) / attachment_name
-        if local_filepath.is_file():
-            attachment_checksum = _uuid if attr.is_external else hash.uuid_from_buffer(data)
-            if attachment_checksum == hash.uuid_from_file(local_filepath, init_string=attachment_name + "\0"):
-                return adapt(str(local_filepath))  # checksum passed, no need to download again
-            # generate the next available alias filename
-            for n in itertools.count():
-                f = local_filepath.parent / (local_filepath.stem + "_%04x" % n + local_filepath.suffix)
-                if not f.is_file():
-                    local_filepath = f
-                    break
-                if attachment_checksum == hash.uuid_from_file(f, init_string=attachment_name + "\0"):
-                    return adapt(str(f))  # checksum passed, no need to download again
-        # Save attachment
-        if attr.is_external:
-            extern.download_attachment(_uuid, attachment_name, local_filepath)
-        else:
-            # write from buffer
-            safe_write(local_filepath, data.split(b"\0", 1)[1])
-        return adapt(str(local_filepath))  # download file from remote store
+        parsed = json.loads(data)
+        if attr.adapter:
+            return attr.adapter.decode(parsed, key=None)
+        return parsed
 
+    # UUID type - convert bytes to UUID object
     if attr.uuid:
-        return adapt(uuid.UUID(bytes=data))
-    elif attr.is_blob:
-        blob_data = extern.get(uuid.UUID(bytes=data)) if attr.is_external else data
-        # Adapters (like <djblob>) handle deserialization in decode()
-        # Without adapter, blob columns return raw bytes (no deserialization)
+        result = uuid_module.UUID(bytes=data)
         if attr.adapter:
-            return attr.adapter.decode(blob_data, key=None)
-        return blob_data  # raw bytes
-    else:
-        return adapt(data)
+            return attr.adapter.decode(result, key=None)
+        return result
+
+    # Blob type - return raw bytes or decode via adapter
+    if attr.is_blob:
+        if attr.adapter:
+            return attr.adapter.decode(data, key=None)
+        return data  # raw bytes
+
+    # Other types with adapter
+    if attr.adapter:
+        return attr.adapter.decode(data, key=None)
+
+    # Native types - pass through unchanged
+    return data
 
 
 class Fetch: