Add Gemini provider, custom URLs, local providers, and extra headers

Makefile

@@ -13,9 +13,11 @@ OBJS = src/pgedge_vectorizer.o \
        src/hybrid_chunking.o \
        src/tokenizer.o \
        src/provider.o \
+       src/provider_common.o \
        src/provider_openai.o \
        src/provider_voyage.o \
        src/provider_ollama.o \
+       src/provider_gemini.o \
        src/worker.o \
        src/queue.o \
        src/embed.o

TODO.md

@@ -4,14 +4,3 @@
 to be assessed as part of the initial work, and inclusion here does not 
 represent a roadmap or any guarantee that any features will actually be
 implemented.
-
-## LLM Support
-
-- Add support for custom base URLs for LLM providers.
-- Add support for use of Google Gemini as the LLM provider.
-- Add support for use of OpenAI API-compatible local LLM providers, such as
-    LM Studio, Docker Model Runner, and EXO. These should just work if 
-    configured as OpenAI, but without a requirement for an API key.
-- Add support for arbitrary request headers to be added to LLM request calls
-    to support servers such as Portkey which requires addition of headers such
-    as `x-portkey-provider: openai`

docs/configuration.md

@@ -8,10 +8,11 @@ These settings configure the connection to your embedding provider, including th
 
 | Parameter | Default | Description | Reload | Restart | Superuser |
 |-----------|---------|-------------|--------|---------|-----------|
-| `pgedge_vectorizer.provider` | `openai` | Embedding provider (openai, voyage, ollama) | No | No | No |
-| `pgedge_vectorizer.api_key_file` | `~/.pgedge-vectorizer-llm-api-key` | API key file path (not needed for Ollama) | No | No | No |
-| `pgedge_vectorizer.api_url` | `https://api.openai.com/v1` | API endpoint | No | No | No |
+| `pgedge_vectorizer.provider` | `openai` | Embedding provider (openai, voyage, ollama, gemini) | No | No | No |
+| `pgedge_vectorizer.api_key_file` | `~/.pgedge-vectorizer-llm-api-key` | API key file path (not needed for Ollama; optional for OpenAI with custom URL) | No | No | No |
+| `pgedge_vectorizer.api_url` | (empty) | API endpoint URL. Leave empty for provider defaults. Set for custom/local endpoints. | No | No | No |
 | `pgedge_vectorizer.model` | `text-embedding-3-small` | Model name | No | No | No |
+| `pgedge_vectorizer.extra_headers` | (empty) | Semicolon-separated `key: value` HTTP headers added to all API requests | No | No | No |
 
 ## Worker Settings
 

docs/embedding_providers.md

@@ -9,7 +9,6 @@ OpenAI provides industry-leading embedding models; these models offer excellent
 **Configuration:**
 ```ini
 pgedge_vectorizer.provider = 'openai'
-pgedge_vectorizer.api_url = 'https://api.openai.com/v1'
 pgedge_vectorizer.api_key_file = '/path/to/api-key-file'
 pgedge_vectorizer.model = 'text-embedding-3-small'
 ```
@@ -19,14 +18,28 @@ pgedge_vectorizer.model = 'text-embedding-3-small'
 - `text-embedding-3-large` - 3072 dimensions, higher quality
 - `text-embedding-ada-002` - 1536 dimensions, legacy model
 
+## Google Gemini
+
+Google Gemini provides embedding models through the Generative Language API.  Save your Gemini API key from https://aistudio.google.com/apikey; specify the path in the `pgedge_vectorizer.api_key_file` configuration parameter.
+
+**Configuration:**
+```ini
+pgedge_vectorizer.provider = 'gemini'
+pgedge_vectorizer.api_key_file = '/path/to/api-key-file'
+pgedge_vectorizer.model = 'text-embedding-004'
+```
+
+**Available Models:**
+- `text-embedding-004` - 768 dimensions, latest model
+- `embedding-001` - 768 dimensions, earlier model
+
 ## Voyage AI
 
 Voyage AI provides high-quality embeddings optimized for retrieval tasks. The API is OpenAI-compatible, making it easy to switch between providers.  Save your Voyage AI API key from https://www.voyageai.com/; specify the path in the `pgedge_vectorizer.api_key_file` configuration parameter.
 
 **Configuration:**
 ```ini
 pgedge_vectorizer.provider = 'voyage'
-pgedge_vectorizer.api_url = 'https://api.voyageai.com/v1'
 pgedge_vectorizer.api_key_file = '/path/to/api-key-file'
 pgedge_vectorizer.model = 'voyage-2'
 ```
@@ -43,7 +56,6 @@ Ollama allows you to run embedding models locally without API keys or internet c
 **Configuration:**
 ```ini
 pgedge_vectorizer.provider = 'ollama'
-pgedge_vectorizer.api_url = 'http://localhost:11434'
 pgedge_vectorizer.model = 'nomic-embed-text'
 # No API key needed for Ollama
 ```
@@ -59,3 +71,40 @@ ollama pull nomic-embed-text
 ```
 
 Note that Ollama doesn't support batch processing, so each text is embedded individually.
+
+## OpenAI-Compatible Local Providers
+
+The OpenAI provider also works with OpenAI-compatible local inference servers. Set `pgedge_vectorizer.api_url` to point at a local server. The API key is optional when using a custom base URL.
+
+**Configuration:**
+```ini
+pgedge_vectorizer.provider = 'openai'
+pgedge_vectorizer.api_url = 'http://localhost:1234/v1'
+pgedge_vectorizer.model = 'your-local-model'
+# No API key needed for local providers
+```
+
+**Compatible Servers and Default URLs:**
+
+| Server | Default URL |
+|--------|------------|
+| Docker Model Runner | `http://localhost:12434/engines/llama.cpp/v1` |
+| llama.cpp | `http://localhost:8080/v1` |
+| LM Studio | `http://localhost:1234/v1` |
+| EXO | `http://localhost:52415/v1` |
+
+## Custom Base URLs
+
+All providers support custom base URLs via the `pgedge_vectorizer.api_url` parameter. When left empty (the default), each provider uses its own default URL. Set a custom URL to use a proxy, gateway, or alternative endpoint.
+
+## Extra Headers
+
+For proxy servers and gateways that require additional HTTP headers (e.g., Portkey), use the `pgedge_vectorizer.extra_headers` parameter:
+
+```ini
+pgedge_vectorizer.provider = 'openai'
+pgedge_vectorizer.api_url = 'https://api.portkey.ai/v1'
+pgedge_vectorizer.extra_headers = 'x-portkey-api-key: pk-xxx; x-portkey-provider: openai'
+```
+
+Headers are semicolon-separated `key: value` pairs.

src/guc.c

@@ -20,6 +20,7 @@ char *pgedge_vectorizer_provider = NULL;
 char *pgedge_vectorizer_api_key_file = NULL;
 char *pgedge_vectorizer_api_url = NULL;
 char *pgedge_vectorizer_model = NULL;
+char *pgedge_vectorizer_extra_headers = NULL;
 
 /*
  * GUC Variables - Worker Configuration
@@ -59,7 +60,7 @@ pgedge_vectorizer_init_guc(void)
 {
 	/* Provider configuration */
 	DefineCustomStringVariable("pgedge_vectorizer.provider",
-								"Embedding provider to use (openai, voyage, ollama)",
+								"Embedding provider to use (openai, voyage, ollama, gemini)",
 								"Determines which API provider is used for generating embeddings.",
 								&pgedge_vectorizer_provider,
 								"openai",
@@ -78,11 +79,16 @@ pgedge_vectorizer_init_guc(void)
 								NULL, NULL, NULL);
 
 	DefineCustomStringVariable("pgedge_vectorizer.api_url",
-								"API endpoint URL",
-								"API endpoint URL. Defaults: OpenAI=https://api.openai.com/v1, "
-								"Voyage=https://api.voyageai.com/v1, Ollama=http://localhost:11434",
+								"API endpoint URL (empty = provider default)",
+								"API endpoint URL. Leave empty to use provider defaults: "
+								"OpenAI=https://api.openai.com/v1, "
+								"Voyage=https://api.voyageai.com/v1, "
+								"Gemini=https://generativelanguage.googleapis.com/v1beta, "
+								"Ollama=http://localhost:11434. "
+								"Set a custom URL for OpenAI-compatible local providers "
+								"(LM Studio, Docker Model Runner, EXO) or proxies (Portkey).",
 								&pgedge_vectorizer_api_url,
-								"https://api.openai.com/v1",
+								"",
 								PGC_USERSET,
 								0,
 								NULL, NULL, NULL);
@@ -99,6 +105,17 @@ pgedge_vectorizer_init_guc(void)
 								0,
 								NULL, NULL, NULL);
 
+	DefineCustomStringVariable("pgedge_vectorizer.extra_headers",
+								"Extra HTTP headers for API requests",
+								"Semicolon-separated key: value pairs added to all "
+								"provider HTTP requests. Example: "
+								"'x-portkey-provider: openai; x-custom: value'",
+								&pgedge_vectorizer_extra_headers,
+								"",
+								PGC_USERSET,
+								0,
+								NULL, NULL, NULL);
+
 	/* Worker configuration */
 	DefineCustomStringVariable("pgedge_vectorizer.databases",
 								"Comma-separated list of databases to monitor",

src/pgedge_vectorizer.h

@@ -50,6 +50,7 @@ extern char *pgedge_vectorizer_provider;
 extern char *pgedge_vectorizer_api_key_file;
 extern char *pgedge_vectorizer_api_url;
 extern char *pgedge_vectorizer_model;
+extern char *pgedge_vectorizer_extra_headers;
 extern char *pgedge_vectorizer_databases;
 extern int pgedge_vectorizer_num_workers;
 extern int pgedge_vectorizer_batch_size;
@@ -163,6 +164,9 @@ extern EmbeddingProvider VoyageProvider;
 /* provider_ollama.c */
 extern EmbeddingProvider OllamaProvider;
 
+/* provider_gemini.c */
+extern EmbeddingProvider GeminiProvider;
+
 /* tokenizer.c */
 int count_tokens(const char *text, const char *model);
 int *tokenize_text(const char *text, const char *model, int *token_count);

src/provider.c

@@ -18,6 +18,7 @@ static EmbeddingProvider *providers[] = {
 	&OpenAIProvider,
 	&VoyageProvider,
 	&OllamaProvider,
+	&GeminiProvider,
 	NULL  /* Sentinel */
 };