diff --git a/changelog/unreleased/SOLR-18152-add-configset-download-zip-to-solrj-fix-schema-designer-bug.yml b/changelog/unreleased/SOLR-18152-add-configset-download-zip-to-solrj-fix-schema-designer-bug.yml new file mode 100644 index 000000000000..68a7cec731e6 --- /dev/null +++ b/changelog/unreleased/SOLR-18152-add-configset-download-zip-to-solrj-fix-schema-designer-bug.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: The "analyze" existing documents feature of Schema Designer was fixed. Added a new ConfigSet.Download and ConfigSet.GetFile capablities to SolrJ. +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Eric Pugh +links: + - name: SOLR-18152 + url: https://issues.apache.org/jira/browse/SOLR-18152 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java index 4bc812043e9d..ac05da1225e5 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java @@ -16,7 +16,11 @@ */ package org.apache.solr.client.api.endpoint; +import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY; + import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.extensions.Extension; +import io.swagger.v3.oas.annotations.extensions.ExtensionProperty; import io.swagger.v3.oas.annotations.parameters.RequestBody; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; @@ -24,10 +28,14 @@ import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; import org.apache.solr.client.api.model.CloneConfigsetRequestBody; +import org.apache.solr.client.api.model.ConfigSetFileContentsResponse; import org.apache.solr.client.api.model.ListConfigsetsResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; @@ -71,6 +79,46 @@ SolrJerseyResponse deleteConfigSet(@PathParam("configSetName") String configSetN throws Exception; } + /** + * V2 API definition for downloading an existing configset as a ZIP archive. + * + *

Equivalent to GET /api/configsets/{configSetName}/download + */ + @Path("/configsets/{configSetName}") + interface Download { + @GET + @Path("/download") + @Operation( + summary = "Download a configset as a ZIP archive.", + tags = {"configsets"}, + extensions = { + @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")}) + }) + @Produces("application/zip") + Response downloadConfigSet( + @PathParam("configSetName") String configSetName, + @QueryParam("displayName") String displayName) + throws Exception; + } + + /** + * V2 API definition for reading a single file from an existing configset. + * + *

Equivalent to GET /api/configsets/{configSetName}/file?path=... + */ + @Path("/configsets/{configSetName}") + interface GetFile { + @GET + @Path("/file") + @Produces(MediaType.TEXT_PLAIN) + @Operation( + summary = "Get the contents of a file in a configset.", + tags = {"configsets"}) + ConfigSetFileContentsResponse getConfigSetFile( + @PathParam("configSetName") String configSetName, @QueryParam("path") String filePath) + throws Exception; + } + /** * V2 API definitions for uploading a configset, in whole or part. * diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/SchemaDesignerApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/SchemaDesignerApi.java new file mode 100644 index 000000000000..713d529703ea --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/SchemaDesignerApi.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import java.util.List; +import org.apache.solr.client.api.model.FlexibleSolrJerseyResponse; +import org.apache.solr.client.api.model.SchemaDesignerCollectionsResponse; +import org.apache.solr.client.api.model.SchemaDesignerConfigsResponse; +import org.apache.solr.client.api.model.SchemaDesignerInfoResponse; +import org.apache.solr.client.api.model.SchemaDesignerPublishResponse; +import org.apache.solr.client.api.model.SchemaDesignerResponse; +import org.apache.solr.client.api.model.SchemaDesignerSchemaDiffResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +/** V2 API definitions for the Solr Schema Designer. */ +@Path("/schema-designer") +public interface SchemaDesignerApi { + + @GET + @Path("/{configSet}/info") + @Operation( + summary = "Get info about a configSet being designed.", + tags = {"schema-designer"}) + SchemaDesignerInfoResponse getInfo(@PathParam("configSet") String configSet) throws Exception; + + @POST + @Path("/{configSet}/prep") + @Operation( + summary = "Prepare a mutable configSet copy for schema design.", + tags = {"schema-designer"}) + SchemaDesignerResponse prepNewSchema( + @PathParam("configSet") String configSet, @QueryParam("copyFrom") String copyFrom) + throws Exception; + + @DELETE + @Path("/{configSet}") + @Operation( + summary = "Clean up temporary resources for a schema being designed.", + tags = {"schema-designer"}) + SolrJerseyResponse cleanupTempSchema(@PathParam("configSet") String configSet) throws Exception; + + @PUT + @Path("/{configSet}/file") + @Operation( + summary = "Update the contents of a file in a configSet being designed.", + tags = {"schema-designer"}) + SchemaDesignerResponse updateFileContents( + @PathParam("configSet") String configSet, @QueryParam("file") String file) throws Exception; + + @GET + @Path("/{configSet}/sample") + @Operation( + summary = "Get a sample value and analysis for a field.", + tags = {"schema-designer"}) + FlexibleSolrJerseyResponse getSampleValue( + @PathParam("configSet") String configSet, + @QueryParam("field") String fieldName, + @QueryParam("uniqueKeyField") String idField, + @QueryParam("docId") String docId) + throws Exception; + + @GET + @Path("/{configSet}/collectionsForConfig") + @Operation( + summary = "List collections that use a given configSet.", + tags = {"schema-designer"}) + SchemaDesignerCollectionsResponse listCollectionsForConfig( + @PathParam("configSet") String configSet) throws Exception; + + @GET + @Path("/configs") + @Operation( + summary = "List all configSets available for schema design.", + tags = {"schema-designer"}) + SchemaDesignerConfigsResponse listConfigs() throws Exception; + + @POST + @Path("/{configSet}/add") + @Operation( + summary = "Add a new field, field type, or dynamic field to the schema being designed.", + tags = {"schema-designer"}) + SchemaDesignerResponse addSchemaObject( + @PathParam("configSet") String configSet, @QueryParam("schemaVersion") Integer schemaVersion) + throws Exception; + + @PUT + @Path("/{configSet}/update") + @Operation( + summary = "Update an existing field or field type in the schema being designed.", + tags = {"schema-designer"}) + SchemaDesignerResponse updateSchemaObject( + @PathParam("configSet") String configSet, @QueryParam("schemaVersion") Integer schemaVersion) + throws Exception; + + @PUT + @Path("/{configSet}/publish") + @Operation( + summary = "Publish the designed schema to a live configSet.", + tags = {"schema-designer"}) + SchemaDesignerPublishResponse publish( + @PathParam("configSet") String configSet, + @QueryParam("schemaVersion") Integer schemaVersion, + @QueryParam("newCollection") String newCollection, + @QueryParam("reloadCollections") @DefaultValue("false") Boolean reloadCollections, + @QueryParam("numShards") @DefaultValue("1") Integer numShards, + @QueryParam("replicationFactor") @DefaultValue("1") Integer replicationFactor, + @QueryParam("indexToCollection") @DefaultValue("false") Boolean indexToCollection, + @QueryParam("cleanupTemp") @DefaultValue("true") Boolean cleanupTempParam, + @QueryParam("disableDesigner") @DefaultValue("false") Boolean disableDesigner) + throws Exception; + + @POST + @Path("/{configSet}/analyze") + @Operation( + summary = "Analyze sample documents and suggest a schema.", + tags = {"schema-designer"}) + SchemaDesignerResponse analyze( + @PathParam("configSet") String configSet, + @QueryParam("schemaVersion") Integer schemaVersion, + @QueryParam("copyFrom") String copyFrom, + @QueryParam("uniqueKeyField") String uniqueKeyField, + @QueryParam("languages") List languages, + @QueryParam("enableDynamicFields") Boolean enableDynamicFields, + @QueryParam("enableFieldGuessing") Boolean enableFieldGuessing, + @QueryParam("enableNestedDocs") Boolean enableNestedDocs) + throws Exception; + + @GET + @Path("/{configSet}/query") + @Operation( + summary = "Query the temporary collection used during schema design.", + tags = {"schema-designer"}) + FlexibleSolrJerseyResponse query(@PathParam("configSet") String configSet) throws Exception; + + @GET + @Path("/{configSet}/diff") + @Operation( + summary = "Get the diff between the designed schema and the published schema.", + tags = {"schema-designer"}) + SchemaDesignerSchemaDiffResponse getSchemaDiff(@PathParam("configSet") String configSet) + throws Exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ConfigSetFileContentsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ConfigSetFileContentsResponse.java new file mode 100644 index 000000000000..3ef99a13bbe1 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ConfigSetFileContentsResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Response type for the "get configset file contents" API. */ +public class ConfigSetFileContentsResponse extends SolrJerseyResponse { + + /** The path of the file within the configset (as requested). */ + @JsonProperty("path") + public String path; + + /** The UTF-8 text content of the file. */ + @JsonProperty("content") + public String content; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerCollectionsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerCollectionsResponse.java new file mode 100644 index 000000000000..2e0d31a27243 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerCollectionsResponse.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +/** Response body for the Schema Designer list-collections-for-config endpoint. */ +public class SchemaDesignerCollectionsResponse extends SolrJerseyResponse { + + @JsonProperty("collections") + public List collections; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerConfigsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerConfigsResponse.java new file mode 100644 index 000000000000..8f8025980822 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerConfigsResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; + +/** Response body for the Schema Designer list-configs endpoint. */ +public class SchemaDesignerConfigsResponse extends SolrJerseyResponse { + + /** + * Map of configSet name to status: 0 = in-progress (temp only), 1 = disabled, 2 = enabled and + * published. + */ + @JsonProperty("configSets") + public Map configSets; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerInfoResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerInfoResponse.java new file mode 100644 index 000000000000..104495af07da --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerInfoResponse.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +/** Response body for the Schema Designer get-info endpoint. */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SchemaDesignerInfoResponse extends SolrJerseyResponse { + + @JsonProperty("configSet") + public String configSet; + + /** Whether the configSet has a published (live) version. */ + @JsonProperty("published") + public Boolean published; + + @JsonProperty("schemaVersion") + public Integer schemaVersion; + + /** Collections currently using this configSet. */ + @JsonProperty("collections") + public List collections; + + /** Number of sample documents stored for this configSet, if available. */ + @JsonProperty("numDocs") + public Integer numDocs; + + // --- designer settings --- + + @JsonProperty("languages") + public List languages; + + @JsonProperty("enableFieldGuessing") + public Boolean enableFieldGuessing; + + @JsonProperty("enableDynamicFields") + public Boolean enableDynamicFields; + + @JsonProperty("enableNestedDocs") + public Boolean enableNestedDocs; + + @JsonProperty("disabled") + public Boolean disabled; + + @JsonProperty("publishedVersion") + public Integer publishedVersion; + + @JsonProperty("copyFrom") + public String copyFrom; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerPublishResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerPublishResponse.java new file mode 100644 index 000000000000..b6d7038ff2fa --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerPublishResponse.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Response body for the Schema Designer publish endpoint. */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SchemaDesignerPublishResponse extends SolrJerseyResponse { + + @JsonProperty("configSet") + public String configSet; + + @JsonProperty("schemaVersion") + public Integer schemaVersion; + + /** The new collection created during publish, if requested. */ + @JsonProperty("newCollection") + public String newCollection; + + /** Error message if indexing sample docs into the new collection failed. */ + @JsonProperty("updateError") + public String updateError; + + @JsonProperty("updateErrorCode") + public Integer updateErrorCode; + + @JsonProperty("errorDetails") + public Object errorDetails; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerResponse.java new file mode 100644 index 000000000000..d41f79d64bb5 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerResponse.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; + +/** + * Response body for Schema Designer endpoints that operate on a full schema: {@code prepNewSchema}, + * {@code updateFileContents}, {@code addSchemaObject}, {@code updateSchemaObject}, and {@code + * analyze}. + * + *

All nullable fields are omitted from JSON output when null. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SchemaDesignerResponse extends SolrJerseyResponse { + + // --- core schema identification --- + + @JsonProperty("configSet") + public String configSet; + + @JsonProperty("schemaVersion") + public Integer schemaVersion; + + /** The temporary mutable collection used during design (e.g. {@code ._designer_myConfig}). */ + @JsonProperty("tempCollection") + public String tempCollection; + + /** Active replica core name for the temp collection, used for Luke API calls. */ + @JsonProperty("core") + public String core; + + @JsonProperty("uniqueKeyField") + public String uniqueKeyField; + + /** Collections currently using the published version of this configSet. */ + @JsonProperty("collectionsForConfig") + public List collectionsForConfig; + + // --- schema objects --- + + @JsonProperty("fields") + public List> fields; + + @JsonProperty("dynamicFields") + public List> dynamicFields; + + @JsonProperty("fieldTypes") + public List> fieldTypes; + + /** ConfigSet files available in ZooKeeper (excluding managed-schema and internal files). */ + @JsonProperty("files") + public List files; + + /** IDs of the first 100 sample documents (present when docs were loaded/analyzed). */ + @JsonProperty("docIds") + public List docIds; + + /** Total number of sample documents, or -1 when no docs were passed to the endpoint. */ + @JsonProperty("numDocs") + public Integer numDocs; + + // --- designer settings --- + + @JsonProperty("languages") + public List languages; + + @JsonProperty("enableFieldGuessing") + public Boolean enableFieldGuessing; + + @JsonProperty("enableDynamicFields") + public Boolean enableDynamicFields; + + @JsonProperty("enableNestedDocs") + public Boolean enableNestedDocs; + + @JsonProperty("disabled") + public Boolean disabled; + + @JsonProperty("publishedVersion") + public Integer publishedVersion; + + @JsonProperty("copyFrom") + public String copyFrom; + + // --- error fields (set when sample-doc indexing fails) --- + + @JsonProperty("updateError") + public String updateError; + + @JsonProperty("updateErrorCode") + public Integer updateErrorCode; + + @JsonProperty("errorDetails") + public Object errorDetails; + + // --- endpoint-specific fields --- + + /** Source of the sample documents (e.g. "blob", "request"); set by {@code analyze}. */ + @JsonProperty("sampleSource") + public String sampleSource; + + /** Analysis warning when field-type inference produced errors; set by {@code analyze}. */ + @JsonProperty("analysisError") + public String analysisError; + + /** + * The type of schema object that was updated: {@code "field"} or {@code "type"}; set by {@code + * updateSchemaObject}. + */ + @JsonProperty("updateType") + public String updateType; + + /** + * The updated field definition map; populated when {@code updateType} is {@code "field"} in + * {@code updateSchemaObject}, or the field name string when returned by {@code addSchemaObject}. + */ + @JsonProperty("field") + public Object field; + + /** + * The updated field-type definition map; populated when {@code updateType} is {@code "type"} in + * {@code updateSchemaObject}, or the type name string when returned by {@code addSchemaObject}. + */ + @JsonProperty("type") + public Object type; + + /** The added dynamic-field name; set by {@code addSchemaObject} when adding a dynamic field. */ + @JsonProperty("dynamicField") + public Object dynamicField; + + /** The added field-type name; set by {@code addSchemaObject} when adding a field type. */ + @JsonProperty("fieldType") + public Object fieldType; + + /** + * Whether the temp collection needs to be rebuilt after this update; set by {@code + * updateSchemaObject}. + */ + @JsonProperty("rebuild") + public Boolean rebuild; + + /** + * Error message when a file update (e.g. {@code solrconfig.xml}) fails validation; set by {@code + * updateFileContents}. + */ + @JsonProperty("updateFileError") + public String updateFileError; + + /** + * The raw file content returned when a file update fails validation; set by {@code + * updateFileContents} so the UI can display the attempted content alongside the error. + */ + @JsonProperty("fileContent") + public String fileContent; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerSchemaDiffResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerSchemaDiffResponse.java new file mode 100644 index 000000000000..f5b9f61d7de0 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaDesignerSchemaDiffResponse.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; + +/** Response body for the Schema Designer get-schema-diff endpoint. */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SchemaDesignerSchemaDiffResponse extends SolrJerseyResponse { + + /** The list of field-level differences between the designed schema and the source. */ + @JsonProperty("diff") + public Map diff; + + /** The configSet used as the diff source (either the published configSet or copyFrom). */ + @JsonProperty("diff-source") + public String diffSource; + + // --- designer settings (reflected from the mutable configSet) --- + + @JsonProperty("languages") + public List languages; + + @JsonProperty("enableFieldGuessing") + public Boolean enableFieldGuessing; + + @JsonProperty("enableDynamicFields") + public Boolean enableDynamicFields; + + @JsonProperty("enableNestedDocs") + public Boolean enableNestedDocs; + + @JsonProperty("disabled") + public Boolean disabled; + + @JsonProperty("publishedVersion") + public Integer publishedVersion; + + @JsonProperty("copyFrom") + public String copyFrom; +} diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 9ca9b377cb2e..f6e85b7273c3 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -126,7 +126,7 @@ import org.apache.solr.handler.admin.ZookeeperStatusHandler; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.handler.component.ShardHandlerFactory; -import org.apache.solr.handler.designer.SchemaDesignerAPI; +import org.apache.solr.handler.designer.SchemaDesigner; import org.apache.solr.jersey.InjectionFactories; import org.apache.solr.jersey.JerseyAppHandlerCache; import org.apache.solr.logging.LogWatcher; @@ -870,7 +870,7 @@ private void loadInternal() { registerV2ApiIfEnabled(clusterAPI.commands); if (isZooKeeperAware()) { - registerV2ApiIfEnabled(new SchemaDesignerAPI(this)); + registerV2ApiIfEnabled(SchemaDesigner.class); } // else Schema Designer not available in standalone (non-cloud) mode /* diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java index edcdc0b1088b..6180a5398309 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java @@ -39,6 +39,8 @@ import org.apache.solr.handler.configsets.CloneConfigSet; import org.apache.solr.handler.configsets.ConfigSetAPIBase; import org.apache.solr.handler.configsets.DeleteConfigSet; +import org.apache.solr.handler.configsets.DownloadConfigSet; +import org.apache.solr.handler.configsets.GetConfigSetFile; import org.apache.solr.handler.configsets.ListConfigSets; import org.apache.solr.handler.configsets.UploadConfigSet; import org.apache.solr.request.SolrQueryRequest; @@ -187,7 +189,12 @@ public Collection getApis() { @Override public Collection> getJerseyResources() { return List.of( - ListConfigSets.class, CloneConfigSet.class, DeleteConfigSet.class, UploadConfigSet.class); + ListConfigSets.class, + CloneConfigSet.class, + DeleteConfigSet.class, + UploadConfigSet.class, + DownloadConfigSet.class, + GetConfigSetFile.class); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/CloneConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/CloneConfigSet.java index c98442d8b44a..b21c4de8056b 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/CloneConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/CloneConfigSet.java @@ -34,7 +34,11 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -/** V2 API implementation for ConfigsetsApi.Clone */ +/** + * V2 API implementation for creating a new configset form an existing one. + * + *

This API (GET /v2/configsets) is analogous to the v1 /admin/configs?action=CREATE command. + */ public class CloneConfigSet extends ConfigSetAPIBase implements ConfigsetsApi.Clone { @Inject diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/DeleteConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/DeleteConfigSet.java index 1a4b363a8339..3b26c5e2fc2b 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/DeleteConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/DeleteConfigSet.java @@ -32,7 +32,11 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -/** V2 API implementation for ConfigsetsApi.Delete */ +/** + * V2 API implementation for deleting a configset + * + *

This API (GET /v2/configsets) is analogous to the v1 /admin/configs?action=DELETE command. + */ public class DeleteConfigSet extends ConfigSetAPIBase implements ConfigsetsApi.Delete { @Inject diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java new file mode 100644 index 000000000000..97de78113443 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.configsets; + +import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM; + +import jakarta.inject.Inject; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.commons.io.file.PathUtils; +import org.apache.solr.client.api.endpoint.ConfigsetsApi; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.core.ConfigSetService; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** V2 API implementation for downloading a configset as a zip file. */ +public class DownloadConfigSet extends ConfigSetAPIBase implements ConfigsetsApi.Download { + + @Inject + public DownloadConfigSet( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + } + + @Override + @PermissionName(CONFIG_READ_PERM) + public Response downloadConfigSet(String configSetName, String displayName) throws Exception { + if (StrUtils.isNullOrEmpty(configSetName)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No configset name provided to download"); + } + if (!configSetService.checkConfigExists(configSetName)) { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, "ConfigSet " + configSetName + " not found!"); + } + final String resolvedDisplayName = + StrUtils.isNullOrEmpty(displayName) ? configSetName : displayName; + return buildZipResponse(configSetService, configSetName, resolvedDisplayName); + } + + /** + * Build a ZIP download {@link Response} for the given configset. + * + * @param configSetService the service to use for downloading the configset files + * @param configSetId the internal configset name to download (may differ from displayName, e.g. + * for schema-designer's mutable copies) + * @param displayName the user-visible name used to derive the download filename + */ + public static Response buildZipResponse( + ConfigSetService configSetService, String configSetId, String displayName) + throws IOException { + final byte[] zipBytes = zipConfigSet(configSetService, configSetId); + final String safeName = displayName.replaceAll("[^a-zA-Z0-9_\\-.]", "_"); + final String fileName = safeName + "_configset.zip"; + return Response.ok((StreamingOutput) outputStream -> outputStream.write(zipBytes)) + .type("application/zip") + .header("Content-Disposition", "attachment; filename=\"" + fileName + "\"") + .build(); + } + + /** + * Download the named configset from {@link ConfigSetService} and return its contents as a ZIP + * archive byte array. + */ + public static byte[] zipConfigSet(ConfigSetService configSetService, String configSetId) + throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Path tmpDirectory = Files.createTempDirectory("configset-download-"); + try { + configSetService.downloadConfig(configSetId, tmpDirectory); + try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { + Files.walkFileTree( + tmpDirectory, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + String dirName = tmpDirectory.relativize(dir).toString(); + if (!dirName.isEmpty()) { + if (!dirName.endsWith("/")) { + dirName += "/"; + } + zipOut.putNextEntry(new ZipEntry(dirName)); + zipOut.closeEntry(); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + if (!Files.isHidden(file)) { + try (InputStream fis = Files.newInputStream(file)) { + ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString()); + zipOut.putNextEntry(zipEntry); + fis.transferTo(zipOut); + } + } + return FileVisitResult.CONTINUE; + } + }); + } + } finally { + PathUtils.deleteDirectory(tmpDirectory); + } + return baos.toByteArray(); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/GetConfigSetFile.java b/solr/core/src/java/org/apache/solr/handler/configsets/GetConfigSetFile.java new file mode 100644 index 000000000000..4c69e60b0a2c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/configsets/GetConfigSetFile.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.configsets; + +import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM; + +import jakarta.inject.Inject; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.apache.solr.client.api.endpoint.ConfigsetsApi; +import org.apache.solr.client.api.model.ConfigSetFileContentsResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** V2 API implementation for reading the contents of a single file from an existing configset. */ +public class GetConfigSetFile extends ConfigSetAPIBase implements ConfigsetsApi.GetFile { + + @Inject + public GetConfigSetFile( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + } + + @Override + @PermissionName(CONFIG_READ_PERM) + public ConfigSetFileContentsResponse getConfigSetFile(String configSetName, String filePath) + throws Exception { + if (StrUtils.isNullOrEmpty(configSetName)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No configset name provided"); + } + if (StrUtils.isNullOrEmpty(filePath)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No file path provided"); + } + if (!configSetService.checkConfigExists(configSetName)) { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, "ConfigSet '" + configSetName + "' not found"); + } + byte[] data = downloadFileFromConfig(configSetName, filePath); + final var response = instantiateJerseyResponse(ConfigSetFileContentsResponse.class); + response.path = filePath; + response.content = + data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : ""; + return response; + } + + private byte[] downloadFileFromConfig(String configSetName, String filePath) { + try { + final byte[] data = configSetService.downloadFileFromConfig(configSetName, filePath); + if (data == null) { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, + "File '" + filePath + "' not found in configset '" + configSetName + "'"); + } + return data; + } catch (IOException e) { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, + "File '" + filePath + "' not found in configset '" + configSetName + "'", + e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java index d87c38154f5a..24a585c6d6fd 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java @@ -40,6 +40,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * V2 API implementation for uploading a configset as a zip file. + * + *

This API (GET /v2/configsets) is analogous to the v1 /admin/configs?action=UPLOAD command. + */ public class UploadConfigSet extends ConfigSetAPIBase implements ConfigsetsApi.Upload { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java index 4dff58a7ba2b..5d53c82801d1 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java @@ -102,7 +102,7 @@ public SampleDocuments parseDocsFromStream( + MAX_STREAM_SIZE + " bytes is the max upload size for sample documents."); } - // use a byte stream for the parsers in case they need to re-parse using a different strategy + // use a byte stream for the parsers in case they need to reparse using a different strategy // e.g. JSON vs. JSON lines or different CSV strategies ... ContentStreamBase.ByteArrayStream byteStream = new ContentStreamBase.ByteArrayStream(uploadedBytes, fileSource, contentType); @@ -153,7 +153,6 @@ protected List loadCsvDocs( .loadDocs(stream); } - @SuppressWarnings("unchecked") protected List loadJsonLines( ContentStreamBase.ByteArrayStream stream, final int maxDocsToLoad) throws IOException { List> docs = new ArrayList<>(); @@ -161,13 +160,7 @@ protected List loadJsonLines( BufferedReader br = new BufferedReader(r); String line; while ((line = br.readLine()) != null) { - line = line.trim(); - if (!line.isEmpty() && line.startsWith("{") && line.endsWith("}")) { - Object jsonLine = ObjectBuilder.getVal(new JSONParser(line)); - if (jsonLine instanceof Map) { - docs.add((Map) jsonLine); - } - } + parseStringToJson(docs, line); if (maxDocsToLoad > 0 && docs.size() == maxDocsToLoad) { break; } @@ -177,6 +170,19 @@ protected List loadJsonLines( return docs.stream().map(JsonLoader::buildDoc).collect(Collectors.toList()); } + private void parseStringToJson(List> docs, String line) throws IOException { + line = line.trim(); + if (line.startsWith("{") && line.endsWith("}")) { + Object jsonLine = ObjectBuilder.getVal(new JSONParser(line)); + if (jsonLine instanceof Map rawMap) { + // JSON object keys are always Strings; the cast is safe + @SuppressWarnings("unchecked") + Map typedMap = (Map) rawMap; + docs.add(typedMap); + } + } + } + @SuppressWarnings("unchecked") protected List loadJsonDocs( ContentStreamBase.ByteArrayStream stream, final int maxDocsToLoad) throws IOException { @@ -204,7 +210,7 @@ protected List loadJsonDocs( if (lines.length > 1) { for (String line : lines) { line = line.trim(); - if (!line.isEmpty() && line.startsWith("{") && line.endsWith("}")) { + if (line.startsWith("{") && line.endsWith("}")) { isJsonLines = true; break; } @@ -294,17 +300,10 @@ protected List parseXmlDocs(XMLStreamReader parser, final int } } - @SuppressWarnings("unchecked") protected List> loadJsonLines(String[] lines) throws IOException { List> docs = new ArrayList<>(lines.length); for (String line : lines) { - line = line.trim(); - if (!line.isEmpty() && line.startsWith("{") && line.endsWith("}")) { - Object jsonLine = ObjectBuilder.getVal(new JSONParser(line)); - if (jsonLine instanceof Map) { - docs.add((Map) jsonLine); - } - } + parseStringToJson(docs, line); } return docs; } diff --git a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java index 543b7a77af16..963cae662830 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java @@ -170,8 +170,7 @@ public Optional suggestField( throw new IllegalStateException("FieldType '" + fieldTypeName + "' not found in the schema!"); } - Map fieldProps = - guessFieldProps(fieldName, fieldType, sampleValues, isMV, schema); + Map fieldProps = guessFieldProps(fieldName, fieldType, isMV, schema); SchemaField schemaField = schema.newField(fieldName, fieldTypeName, fieldProps); return Optional.of(schemaField); } @@ -179,9 +178,9 @@ public Optional suggestField( @Override public ManagedIndexSchema adaptExistingFieldToData( SchemaField schemaField, List sampleValues, ManagedIndexSchema schema) { - // Promote a single-valued to multi-valued if needed + // Promote a single-valued to multivalued if needed if (!schemaField.multiValued() && isMultiValued(sampleValues)) { - // this existing field needs to be promoted to multi-valued + // this existing field needs to be promoted to multivalued SimpleOrderedMap fieldProps = schemaField.getNamedPropertyValues(false); fieldProps.add("multiValued", true); fieldProps.remove("name"); @@ -210,7 +209,7 @@ public Map> transposeDocs(List docs) { Collection fieldValues = doc.getFieldValues(f); if (fieldValues != null && !fieldValues.isEmpty()) { if (fieldValues.size() == 1) { - // flatten so every field doesn't end up multi-valued + // flatten so every field doesn't end up multivalued values.add(fieldValues.iterator().next()); } else { // truly multi-valued @@ -395,11 +394,7 @@ protected boolean isMultiValued(final List sampleValues) { } protected Map guessFieldProps( - String fieldName, - FieldType fieldType, - List sampleValues, - boolean isMV, - IndexSchema schema) { + String fieldName, FieldType fieldType, boolean isMV, IndexSchema schema) { Map props = new HashMap<>(); props.put("indexed", "true"); diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SampleDocuments.java b/solr/core/src/java/org/apache/solr/handler/designer/SampleDocuments.java index b98c5995db28..6037db515241 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SampleDocuments.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SampleDocuments.java @@ -56,7 +56,7 @@ public List appendDocs( return id != null && !ids.contains(id); // doc has ID, and it's not already in the set }) - .collect(Collectors.toList()); + .toList(); parsed.addAll(toAdd); if (maxDocsToLoad > 0 && parsed.size() > maxDocsToLoad) { parsed = parsed.subList(0, maxDocsToLoad); diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesigner.java similarity index 70% rename from solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java rename to solr/core/src/java/org/apache/solr/handler/designer/SchemaDesigner.java index 91f8e7e57d29..4c776bfcc25e 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesigner.java @@ -17,14 +17,12 @@ package org.apache.solr.handler.designer; -import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; -import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; -import static org.apache.solr.client.solrj.SolrRequest.METHOD.PUT; import static org.apache.solr.common.params.CommonParams.JSON_MIME; import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME; import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM; +import jakarta.inject.Inject; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -49,7 +47,16 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.apache.solr.api.EndPoint; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.SchemaDesignerApi; +import org.apache.solr.client.api.model.FlexibleSolrJerseyResponse; +import org.apache.solr.client.api.model.SchemaDesignerCollectionsResponse; +import org.apache.solr.client.api.model.SchemaDesignerConfigsResponse; +import org.apache.solr.client.api.model.SchemaDesignerInfoResponse; +import org.apache.solr.client.api.model.SchemaDesignerPublishResponse; +import org.apache.solr.client.api.model.SchemaDesignerResponse; +import org.apache.solr.client.api.model.SchemaDesignerSchemaDiffResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -57,6 +64,7 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.ZkConfigSetService; import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; @@ -66,16 +74,14 @@ import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.ContentStream; -import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.RawResponseWriter; -import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.ManagedIndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.util.RTimer; @@ -86,7 +92,8 @@ import org.slf4j.LoggerFactory; /** All V2 APIs have a prefix of /api/schema-designer/ */ -public class SchemaDesignerAPI implements SchemaDesignerConstants { +public class SchemaDesigner extends JerseyResource + implements SchemaDesignerApi, SchemaDesignerConstants { private static final Set excludeConfigSetNames = Set.of(DEFAULT_CONFIGSET_NAME); @@ -98,21 +105,26 @@ public class SchemaDesignerAPI implements SchemaDesignerConstants { private final SchemaDesignerSettingsDAO settingsDAO; private final SchemaDesignerConfigSetHelper configSetHelper; private final Map indexedVersion = new ConcurrentHashMap<>(); + private final SolrQueryRequest solrQueryRequest; - public SchemaDesignerAPI(CoreContainer coreContainer) { + @Inject + public SchemaDesigner(CoreContainer coreContainer, SolrQueryRequest solrQueryRequest) { this( coreContainer, - SchemaDesignerAPI.newSchemaSuggester(), - SchemaDesignerAPI.newSampleDocumentsLoader()); + SchemaDesigner.newSchemaSuggester(), + SchemaDesigner.newSampleDocumentsLoader(), + solrQueryRequest); } - SchemaDesignerAPI( + SchemaDesigner( CoreContainer coreContainer, SchemaSuggester schemaSuggester, - SampleDocumentsLoader sampleDocLoader) { + SampleDocumentsLoader sampleDocLoader, + SolrQueryRequest solrQueryRequest) { this.coreContainer = coreContainer; this.schemaSuggester = schemaSuggester; this.sampleDocLoader = sampleDocLoader; + this.solrQueryRequest = solrQueryRequest; this.configSetHelper = new SchemaDesignerConfigSetHelper(this.coreContainer, this.schemaSuggester); this.settingsDAO = new SchemaDesignerSettingsDAO(coreContainer, configSetHelper); @@ -146,14 +158,16 @@ static String getMutableId(final String configSet) { return DESIGNER_PREFIX + configSet; } - @EndPoint(method = GET, path = "/schema-designer/info", permission = CONFIG_READ_PERM) - public void getInfo(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); + @Override + @PermissionName(CONFIG_READ_PERM) + public SchemaDesignerInfoResponse getInfo(String configSet) throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); - Map responseMap = new HashMap<>(); - responseMap.put(CONFIG_SET_PARAM, configSet); + SchemaDesignerInfoResponse response = + instantiateJerseyResponse(SchemaDesignerInfoResponse.class); + response.configSet = configSet; boolean exists = configExists(configSet); - responseMap.put("published", exists); + response.published = exists; // mutable config may not exist yet as this is just an info check to gather some basic info the // UI needs @@ -164,31 +178,31 @@ public void getInfo(SolrQueryRequest req, SolrQueryResponse rsp) throws IOExcept SolrConfig srcConfig = exists ? configSetHelper.loadSolrConfig(configSet) : null; SolrConfig solrConfig = configExists(mutableId) ? configSetHelper.loadSolrConfig(mutableId) : srcConfig; - addSettingsToResponse(settingsDAO.getSettings(solrConfig), responseMap); + addSettingsToResponse(settingsDAO.getSettings(solrConfig), response); - responseMap.put(SCHEMA_VERSION_PARAM, configSetHelper.getCurrentSchemaVersion(mutableId)); - responseMap.put( - "collections", exists ? configSetHelper.listCollectionsForConfig(configSet) : List.of()); + response.schemaVersion = configSetHelper.getCurrentSchemaVersion(mutableId); + response.collections = exists ? configSetHelper.listCollectionsForConfig(configSet) : List.of(); // don't fail if loading sample docs fails try { - responseMap.put("numDocs", configSetHelper.retrieveSampleDocs(configSet).size()); + response.numDocs = configSetHelper.retrieveSampleDocs(configSet).size(); } catch (Exception exc) { log.warn("Failed to load sample docs from blob store for {}", configSet, exc); } - rsp.getValues().addAll(responseMap); + return response; } - @EndPoint(method = POST, path = "/schema-designer/prep", permission = CONFIG_EDIT_PERM) - public void prepNewSchema(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SchemaDesignerResponse prepNewSchema(String configSet, String copyFrom) throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); validateNewConfigSetName(configSet); - final String copyFrom = req.getParams().get(COPY_FROM_PARAM, DEFAULT_CONFIGSET_NAME); + final String effectiveCopyFrom = copyFrom != null ? copyFrom : DEFAULT_CONFIGSET_NAME; - SchemaDesignerSettings settings = getMutableSchemaForConfigSet(configSet, -1, copyFrom); + SchemaDesignerSettings settings = + getMutableSchemaForConfigSet(configSet, -1, effectiveCopyFrom); ManagedIndexSchema schema = settings.getSchema(); String mutableId = getMutableId(configSet); @@ -200,36 +214,22 @@ public void prepNewSchema(SolrQueryRequest req, SolrQueryResponse rsp) settingsDAO.persistIfChanged(mutableId, settings); - rsp.getValues().addAll(buildResponse(configSet, schema, settings, null)); - } - - @EndPoint(method = PUT, path = "/schema-designer/cleanup", permission = CONFIG_EDIT_PERM) - public void cleanupTemp(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - cleanupTemp(getRequiredParam(CONFIG_SET_PARAM, req)); + return buildSchemaDesignerResponse(configSet, schema, settings, null); } - @EndPoint(method = GET, path = "/schema-designer/file", permission = CONFIG_READ_PERM) - public void getFileContents(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - final String file = getRequiredParam("file", req); - String filePath = getConfigSetZkPath(getMutableId(configSet), file); - byte[] data; - try { - data = zkStateReader().getZkClient().getData(filePath, null, null); - } catch (KeeperException | InterruptedException e) { - throw new IOException("Error reading file: " + filePath, SolrZkClient.checkInterrupted(e)); - } - String stringData = - data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : ""; - rsp.getValues().addAll(Collections.singletonMap(file, stringData)); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SolrJerseyResponse cleanupTempSchema(String configSet) throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); + doCleanupTemp(configSet); + return instantiateJerseyResponse(SolrJerseyResponse.class); } - @EndPoint(method = POST, path = "/schema-designer/file", permission = CONFIG_EDIT_PERM) - public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - final String file = getRequiredParam("file", req); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SchemaDesignerResponse updateFileContents(String configSet, String file) throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); + requireNotEmpty("file", file); String mutableId = getMutableId(configSet); String zkPath = getConfigSetZkPath(mutableId, file); @@ -240,7 +240,7 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp) } byte[] data; - try (InputStream in = extractSingleContentStream(req, true).getStream()) { + try (InputStream in = extractSingleContentStream(true).getStream()) { data = in.readAllBytes(); } Exception updateFileError = null; @@ -261,11 +261,11 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp) // solrconfig.xml update failed, but haven't impacted the configSet yet, so just return the // error directly Throwable causedBy = SolrException.getRootCause(updateFileError); - Map response = new HashMap<>(); - response.put("updateFileError", causedBy.getMessage()); - response.put(file, new String(data, StandardCharsets.UTF_8)); - rsp.getValues().addAll(response); - return; + SchemaDesignerResponse errorResponse = + instantiateJerseyResponse(SchemaDesignerResponse.class); + errorResponse.updateFileError = causedBy.getMessage(); + errorResponse.fileContent = new String(data, StandardCharsets.UTF_8); + return errorResponse; } // apply the update and reload the temp collection / re-index sample docs @@ -295,10 +295,10 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp) } } - Map response = buildResponse(configSet, schema, null, docs); + SchemaDesignerResponse response = buildSchemaDesignerResponse(configSet, schema, null, docs); if (analysisErrorHolder[0] != null) { - response.put(ANALYSIS_ERROR, analysisErrorHolder[0]); + response.analysisError = analysisErrorHolder[0]; } addErrorToResponse( @@ -308,15 +308,16 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp) response, "Failed to re-index sample documents after update to the " + file + " file"); - rsp.getValues().addAll(response); + return response; } - @EndPoint(method = GET, path = "/schema-designer/sample", permission = CONFIG_READ_PERM) - public void getSampleValue(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - final String fieldName = getRequiredParam(FIELD_PARAM, req); - final String idField = getRequiredParam(UNIQUE_KEY_FIELD_PARAM, req); - String docId = req.getParams().get(DOC_ID_PARAM); + @Override + @PermissionName(CONFIG_READ_PERM) + public FlexibleSolrJerseyResponse getSampleValue( + String configSet, String fieldName, String idField, String docId) throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); + requireNotEmpty(FIELD_PARAM, fieldName); + requireNotEmpty(UNIQUE_KEY_FIELD_PARAM, idField); final List docs = configSetHelper.retrieveSampleDocs(configSet); String textValue = null; @@ -347,27 +348,31 @@ public void getSampleValue(SolrQueryRequest req, SolrQueryResponse rsp) throws I if (textValue != null) { var analysis = configSetHelper.analyzeField(configSet, fieldName, textValue); - rsp.getValues().addAll(Map.of(idField, docId, fieldName, textValue, "analysis", analysis)); + return buildFlexibleResponse( + Map.of(idField, docId, fieldName, textValue, "analysis", analysis)); } + return instantiateJerseyResponse(FlexibleSolrJerseyResponse.class); } - @EndPoint( - method = GET, - path = "/schema-designer/collectionsForConfig", - permission = CONFIG_READ_PERM) - public void listCollectionsForConfig(SolrQueryRequest req, SolrQueryResponse rsp) { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - rsp.getValues() - .addAll( - Collections.singletonMap( - "collections", configSetHelper.listCollectionsForConfig(configSet))); + @Override + @PermissionName(CONFIG_READ_PERM) + public SchemaDesignerCollectionsResponse listCollectionsForConfig(String configSet) { + requireNotEmpty(CONFIG_SET_PARAM, configSet); + SchemaDesignerCollectionsResponse response = + instantiateJerseyResponse(SchemaDesignerCollectionsResponse.class); + response.collections = configSetHelper.listCollectionsForConfig(configSet); + return response; } // CONFIG_EDIT_PERM is required here since this endpoint is used by the UI to determine if the // user has access to the Schema Designer UI - @EndPoint(method = GET, path = "/schema-designer/configs", permission = CONFIG_EDIT_PERM) - public void listConfigs(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - rsp.getValues().addAll(Collections.singletonMap("configSets", listEnabledConfigs())); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SchemaDesignerConfigsResponse listConfigs() throws Exception { + SchemaDesignerConfigsResponse response = + instantiateJerseyResponse(SchemaDesignerConfigsResponse.class); + response.configSets = listEnabledConfigs(); + return response; } protected Map listEnabledConfigs() throws IOException { @@ -386,62 +391,38 @@ protected Map listEnabledConfigs() throws IOException { return configs; } - @EndPoint(method = GET, path = "/schema-designer/download/*", permission = CONFIG_READ_PERM) - public void downloadConfig(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - String mutableId = getMutableId(configSet); - - // find the configSet to download - SolrZkClient zkClient = zkStateReader().getZkClient(); - String configId = mutableId; - try { - if (!zkClient.exists(getConfigSetZkPath(mutableId, null))) { - if (zkClient.exists(getConfigSetZkPath(configSet, null))) { - configId = configSet; - } else { - throw new SolrException( - SolrException.ErrorCode.NOT_FOUND, "ConfigSet " + configSet + " not found!"); - } - } - } catch (KeeperException | InterruptedException e) { - throw new IOException("Error reading config from ZK", SolrZkClient.checkInterrupted(e)); - } - - ContentStreamBase content = - new ContentStreamBase.ByteArrayStream( - configSetHelper.downloadAndZipConfigSet(configId), - configSet + ".zip", - "application/zip"); - rsp.add(RawResponseWriter.CONTENT, content); - } - - @EndPoint(method = POST, path = "/schema-designer/add", permission = CONFIG_EDIT_PERM) - public void addSchemaObject(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - final String mutableId = checkMutable(configSet, req); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SchemaDesignerResponse addSchemaObject(String configSet, Integer schemaVersion) + throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); + requireSchemaVersion(schemaVersion); + final String mutableId = checkMutable(configSet, schemaVersion); - Map addJson = readJsonFromRequest(req); + Map addJson = readJsonFromRequest(); log.info("Adding new schema object from JSON: {}", addJson); String objectName = configSetHelper.addSchemaObject(configSet, addJson); String action = addJson.keySet().iterator().next(); ManagedIndexSchema schema = loadLatestSchema(mutableId); - Map response = - buildResponse(configSet, schema, null, configSetHelper.retrieveSampleDocs(configSet)); - response.put(action, objectName); - rsp.getValues().addAll(response); + SchemaDesignerResponse response = + buildSchemaDesignerResponse( + configSet, schema, null, configSetHelper.retrieveSampleDocs(configSet)); + setSchemaObjectField(response, action, objectName); + return response; } - @EndPoint(method = PUT, path = "/schema-designer/update", permission = CONFIG_EDIT_PERM) - public void updateSchemaObject(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - final String mutableId = checkMutable(configSet, req); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SchemaDesignerResponse updateSchemaObject(String configSet, Integer schemaVersion) + throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); + requireSchemaVersion(schemaVersion); + final String mutableId = checkMutable(configSet, schemaVersion); // Updated field definition is in the request body as JSON - Map updateField = readJsonFromRequest(req); + Map updateField = readJsonFromRequest(); String name = (String) updateField.get("name"); if (StrUtils.isNullOrEmpty(name)) { throw new SolrException( @@ -488,29 +469,41 @@ public void updateSchemaObject(SolrQueryRequest req, SolrQueryResponse rsp) } } - Map response = buildResponse(configSet, schema, settings, docs); - response.put("updateType", updateType); + SchemaDesignerResponse response = + buildSchemaDesignerResponse(configSet, schema, settings, docs); + response.updateType = updateType; if (FIELD_PARAM.equals(updateType)) { - response.put(updateType, fieldToMap(schema.getField(name), schema)); + response.field = fieldToMap(schema.getField(name), schema); } else if ("type".equals(updateType)) { - response.put(updateType, schema.getFieldTypeByName(name).getNamedPropertyValues(true)); + response.type = schema.getFieldTypeByName(name).getNamedPropertyValues(true); } if (analysisErrorHolder[0] != null) { - response.put(ANALYSIS_ERROR, analysisErrorHolder[0]); + response.analysisError = analysisErrorHolder[0]; } addErrorToResponse(mutableId, solrExc, errorsDuringIndexing, response, updateError); - response.put("rebuild", needsRebuild); - rsp.getValues().addAll(response); + response.rebuild = needsRebuild; + return response; } - @EndPoint(method = PUT, path = "/schema-designer/publish", permission = CONFIG_EDIT_PERM) - public void publish(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); - final String mutableId = checkMutable(configSet, req); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SchemaDesignerPublishResponse publish( + String configSet, + Integer schemaVersion, + String newCollection, + Boolean reloadCollections, + Integer numShards, + Integer replicationFactor, + Boolean indexToCollection, + Boolean cleanupTempParam, + Boolean disableDesigner) + throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); + requireSchemaVersion(schemaVersion); + final String mutableId = checkMutable(configSet, schemaVersion); // verify the configSet we're going to apply changes to hasn't been changed since being loaded // for @@ -533,7 +526,6 @@ public void publish(SolrQueryRequest req, SolrQueryResponse rsp) } } - String newCollection = req.getParams().get(NEW_COLLECTION_PARAM); if (StrUtils.isNotNullOrEmpty(newCollection) && zkStateReader().getClusterState().hasCollection(newCollection)) { throw new SolrException( @@ -554,7 +546,6 @@ && zkStateReader().getClusterState().hasCollection(newCollection)) { copyConfig(mutableId, configSet); } - boolean reloadCollections = req.getParams().getBool(RELOAD_COLLECTIONS_PARAM, false); if (reloadCollections) { log.debug("Reloading collections after update to configSet: {}", configSet); List collectionsForConfig = configSetHelper.listCollectionsForConfig(configSet); @@ -567,10 +558,8 @@ && zkStateReader().getClusterState().hasCollection(newCollection)) { // create new collection Map errorsDuringIndexing = null; if (StrUtils.isNotNullOrEmpty(newCollection)) { - int numShards = req.getParams().getInt("numShards", 1); - int rf = req.getParams().getInt("replicationFactor", 1); - configSetHelper.createCollection(newCollection, configSet, numShards, rf); - if (req.getParams().getBool(INDEX_TO_COLLECTION_PARAM, false)) { + configSetHelper.createCollection(newCollection, configSet, numShards, replicationFactor); + if (indexToCollection) { List docs = configSetHelper.retrieveSampleDocs(configSet); if (!docs.isEmpty()) { ManagedIndexSchema schema = loadLatestSchema(mutableId); @@ -580,35 +569,45 @@ && zkStateReader().getClusterState().hasCollection(newCollection)) { } } - if (req.getParams().getBool(CLEANUP_TEMP_PARAM, true)) { + if (cleanupTempParam) { try { - cleanupTemp(configSet); + doCleanupTemp(configSet); } catch (IOException | SolrServerException | SolrException exc) { final String excStr = exc.toString(); log.warn("Failed to clean-up temp collection {} due to: {}", mutableId, excStr); } } - settings.setDisabled(req.getParams().getBool(DISABLE_DESIGNER_PARAM, false)); + settings.setDisabled(disableDesigner); settingsDAO.persistIfChanged(configSet, settings); - Map response = new HashMap<>(); - response.put(CONFIG_SET_PARAM, configSet); - response.put(SCHEMA_VERSION_PARAM, configSetHelper.getCurrentSchemaVersion(configSet)); + SchemaDesignerPublishResponse response = + instantiateJerseyResponse(SchemaDesignerPublishResponse.class); + response.configSet = configSet; + response.schemaVersion = configSetHelper.getCurrentSchemaVersion(configSet); if (StrUtils.isNotNullOrEmpty(newCollection)) { - response.put(NEW_COLLECTION_PARAM, newCollection); + response.newCollection = newCollection; } - addErrorToResponse(newCollection, null, errorsDuringIndexing, response, null); + addErrorToResponse(newCollection, null, errorsDuringIndexing, response); - rsp.getValues().addAll(response); + return response; } - @EndPoint(method = POST, path = "/schema-designer/analyze", permission = CONFIG_EDIT_PERM) - public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - final int schemaVersion = req.getParams().getInt(SCHEMA_VERSION_PARAM, -1); - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); + @Override + @PermissionName(CONFIG_EDIT_PERM) + public SchemaDesignerResponse analyze( + String configSet, + Integer schemaVersion, + String copyFrom, + String uniqueKeyField, + List languages, + Boolean enableDynamicFields, + Boolean enableFieldGuessing, + Boolean enableNestedDocs) + throws Exception { + final int schemaVersionInt = schemaVersion != null ? schemaVersion : -1; + requireNotEmpty(CONFIG_SET_PARAM, configSet); // don't let the user edit the _default configSet with the designer (for now) if (DEFAULT_CONFIGSET_NAME.equals(configSet)) { @@ -622,27 +621,25 @@ public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) // Get the sample documents to analyze, preferring those in the request but falling back to // previously stored - SampleDocuments sampleDocuments = loadSampleDocuments(req, configSet); + SampleDocuments sampleDocuments = loadSampleDocuments(configSet); // Get a mutable "temp" schema either from the specified copy source or configSet if it already // exists. - String copyFrom = - configExists(configSet) - ? configSet - : req.getParams().get(COPY_FROM_PARAM, DEFAULT_CONFIGSET_NAME); + if (copyFrom == null) { + copyFrom = configExists(configSet) ? configSet : DEFAULT_CONFIGSET_NAME; + } String mutableId = getMutableId(configSet); // holds additional settings needed by the designer to maintain state SchemaDesignerSettings settings = - getMutableSchemaForConfigSet(configSet, schemaVersion, copyFrom); + getMutableSchemaForConfigSet(configSet, schemaVersionInt, copyFrom); ManagedIndexSchema schema = settings.getSchema(); - String uniqueKeyFieldParam = req.getParams().get(UNIQUE_KEY_FIELD_PARAM); - if (StrUtils.isNotNullOrEmpty(uniqueKeyFieldParam)) { - String uniqueKeyField = + if (StrUtils.isNotNullOrEmpty(uniqueKeyField)) { + String existingKeyField = schema.getUniqueKeyField() != null ? schema.getUniqueKeyField().getName() : null; - if (!uniqueKeyFieldParam.equals(uniqueKeyField)) { + if (!uniqueKeyField.equals(existingKeyField)) { // The Schema API doesn't support changing the ID field so would have to use XML directly throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, @@ -651,13 +648,12 @@ public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) } boolean langsUpdated = false; - String[] languages = req.getParams().getParams(LANGUAGES_PARAM); List langs; if (languages != null) { langs = - languages.length == 0 || (languages.length == 1 && "*".equals(languages[0])) + languages.isEmpty() || (languages.size() == 1 && "*".equals(languages.getFirst())) ? List.of() - : Arrays.asList(languages); + : languages; if (!langs.equals(settings.getLanguages())) { settings.setLanguages(langs); langsUpdated = true; @@ -668,7 +664,6 @@ public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) } boolean dynamicUpdated = false; - Boolean enableDynamicFields = req.getParams().getBool(ENABLE_DYNAMIC_FIELDS_PARAM); if (enableDynamicFields != null && enableDynamicFields != settings.dynamicFieldsEnabled()) { settings.setDynamicFieldsEnabled(enableDynamicFields); dynamicUpdated = true; @@ -699,7 +694,6 @@ public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) // persist the updated schema schema.persistManagedSchema(false); - Boolean enableFieldGuessing = req.getParams().getBool(ENABLE_FIELD_GUESSING_PARAM); if (enableFieldGuessing != null && enableFieldGuessing != settings.fieldGuessingEnabled()) { settings.setFieldGuessingEnabled(enableFieldGuessing); } @@ -714,7 +708,6 @@ public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) } // nested docs - Boolean enableNestedDocs = req.getParams().getBool(ENABLE_NESTED_DOCS_PARAM); if (enableNestedDocs != null && enableNestedDocs != settings.nestedDocsEnabled()) { settings.setNestedDocsEnabled(enableNestedDocs); configSetHelper.toggleNestedDocsFields(schema, enableNestedDocs); @@ -734,20 +727,20 @@ public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) CollectionAdminRequest.reloadCollection(mutableId).process(cloudClient()); } - Map response = - buildResponse(configSet, loadLatestSchema(mutableId), settings, docs); - response.put("sampleSource", sampleDocuments.getSource()); + SchemaDesignerResponse response = + buildSchemaDesignerResponse(configSet, loadLatestSchema(mutableId), settings, docs); + response.sampleSource = sampleDocuments.getSource(); if (analysisErrorHolder[0] != null) { - response.put(ANALYSIS_ERROR, analysisErrorHolder[0]); + response.analysisError = analysisErrorHolder[0]; } addErrorToResponse(mutableId, null, errorsDuringIndexing, response, null); - rsp.getValues().addAll(response); + return response; } - @EndPoint(method = GET, path = "/schema-designer/query", permission = CONFIG_READ_PERM) - public void query(SolrQueryRequest req, SolrQueryResponse rsp) - throws IOException, SolrServerException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); + @Override + @PermissionName(CONFIG_READ_PERM) + public FlexibleSolrJerseyResponse query(String configSet) throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); String mutableId = getMutableId(configSet); if (!configExists(mutableId)) { throw new SolrException( @@ -774,57 +767,80 @@ public void query(SolrQueryRequest req, SolrQueryResponse rsp) version, currentVersion); List docs = configSetHelper.retrieveSampleDocs(configSet); - ManagedIndexSchema schema = loadLatestSchema(mutableId); - errorsDuringIndexing = - indexSampleDocsWithRebuildOnAnalysisError( - schema.getUniqueKeyField().getName(), docs, mutableId, true, null); - // the version changes when you index (due to field guessing URP) - currentVersion = configSetHelper.getCurrentSchemaVersion(mutableId); + if (!docs.isEmpty()) { + ManagedIndexSchema schema = loadLatestSchema(mutableId); + errorsDuringIndexing = + indexSampleDocsWithRebuildOnAnalysisError( + schema.getUniqueKeyField().getName(), docs, mutableId, true, null); + // the version changes when you index (due to field guessing URP) + currentVersion = configSetHelper.getCurrentSchemaVersion(mutableId); + } indexedVersion.put(mutableId, currentVersion); } if (errorsDuringIndexing != null) { - Map response = new HashMap<>(); - rsp.setException( + Map errorResponse = new HashMap<>(); + addErrorToResponse( + mutableId, new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Failed to re-index sample documents after schema updated.")); - response.put(ERROR_DETAILS, errorsDuringIndexing); - rsp.getValues().addAll(response); - return; + "Failed to re-index sample documents after schema updated."), + errorsDuringIndexing, + errorResponse, + "Failed to re-index sample documents after schema updated."); + return buildFlexibleResponse(errorResponse); } // execute the user's query against the temp collection - QueryResponse qr = cloudClient().query(mutableId, req.getParams()); - rsp.getValues().addAll(qr.getResponse()); + QueryResponse qr = cloudClient().query(mutableId, solrQueryRequest.getParams()); + Map responseMap = new HashMap<>(); + qr.getResponse() + .forEach( + (name, val) -> { + if ("response".equals(name) && val instanceof SolrDocumentList) { + // SolrDocumentList extends ArrayList, so Jackson would serialize it as a plain + // array, losing numFound/start metadata that the UI expects at data.response.docs + SolrDocumentList docList = (SolrDocumentList) val; + Map responseObj = new HashMap<>(); + responseObj.put("numFound", docList.getNumFound()); + responseObj.put("start", docList.getStart()); + responseObj.put("docs", new ArrayList<>(docList)); + responseMap.put(name, responseObj); + } else { + responseMap.put(name, val); + } + }); + return buildFlexibleResponse(responseMap); } /** * Return the diff of designer schema with the source schema (either previously published or the * copyFrom). */ - @EndPoint(method = GET, path = "/schema-designer/diff", permission = CONFIG_READ_PERM) - public void getSchemaDiff(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - final String configSet = getRequiredParam(CONFIG_SET_PARAM, req); + @Override + @PermissionName(CONFIG_READ_PERM) + public SchemaDesignerSchemaDiffResponse getSchemaDiff(String configSet) throws Exception { + requireNotEmpty(CONFIG_SET_PARAM, configSet); SchemaDesignerSettings settings = getMutableSchemaForConfigSet(configSet, -1, null); // diff the published if found, else use the original source schema String sourceSchema = configExists(configSet) ? configSet : settings.getCopyFrom(); - Map response = new HashMap<>(); - response.put( - "diff", ManagedSchemaDiff.diff(loadLatestSchema(sourceSchema), settings.getSchema())); - response.put("diff-source", sourceSchema); + SchemaDesignerSchemaDiffResponse response = + instantiateJerseyResponse(SchemaDesignerSchemaDiffResponse.class); + response.diff = ManagedSchemaDiff.diff(loadLatestSchema(sourceSchema), settings.getSchema()); + response.diffSource = sourceSchema; addSettingsToResponse(settings, response); - rsp.getValues().addAll(response); + return response; } - protected SampleDocuments loadSampleDocuments(SolrQueryRequest req, String configSet) - throws IOException { + protected SampleDocuments loadSampleDocuments(String configSet) throws IOException { List docs = null; - ContentStream stream = extractSingleContentStream(req, false); + ContentStream stream = extractSingleContentStream(false); SampleDocuments sampleDocs = null; if (stream != null && stream.getContentType() != null) { - sampleDocs = sampleDocLoader.parseDocsFromStream(req.getParams(), stream, MAX_SAMPLE_DOCS); + sampleDocs = + sampleDocLoader.parseDocsFromStream( + solrQueryRequest.getParams(), stream, MAX_SAMPLE_DOCS); docs = sampleDocs.parsed; if (!docs.isEmpty()) { // user posted in some docs, if there are already docs stored in the blob store, then add @@ -885,7 +901,7 @@ protected ManagedIndexSchema analyzeInputDocs( return schema; } - protected SchemaDesignerSettings getMutableSchemaForConfigSet( + SchemaDesignerSettings getMutableSchemaForConfigSet( final String configSet, final int schemaVersion, String copyFrom) throws IOException { // The designer works with mutable config sets stored in a "temp" znode in ZK instead of the // "live" configSet @@ -965,8 +981,8 @@ ManagedIndexSchema loadLatestSchema(String configSet) { return configSetHelper.loadLatestSchema(configSet); } - protected ContentStream extractSingleContentStream(final SolrQueryRequest req, boolean required) { - Iterable streams = req.getContentStreams(); + protected ContentStream extractSingleContentStream(boolean required) { + Iterable streams = solrQueryRequest.getContentStreams(); Iterator iter = streams != null ? streams.iterator() : null; ContentStream stream = iter != null && iter.hasNext() ? iter.next() : null; if (required && stream == null) @@ -1106,7 +1122,7 @@ protected long waitToSeeSampleDocs(String collectionName, long numAdded) return numFound; } - protected Map buildResponse( + SchemaDesignerResponse buildSchemaDesignerResponse( String configSet, final ManagedIndexSchema schema, SchemaDesignerSettings settings, @@ -1116,50 +1132,44 @@ protected Map buildResponse( int currentVersion = configSetHelper.getCurrentSchemaVersion(mutableId); indexedVersion.put(mutableId, currentVersion); - // response is a map of data structures to support the schema designer - Map response = new HashMap<>(); + SchemaDesignerResponse response = instantiateJerseyResponse(SchemaDesignerResponse.class); DocCollection coll = zkStateReader().getCollection(mutableId); Collection activeSlices = coll.getActiveSlices(); if (!activeSlices.isEmpty()) { - String coreName = activeSlices.stream().findAny().orElseThrow().getLeader().getCoreName(); - response.put("core", coreName); + response.core = activeSlices.stream().findAny().orElseThrow().getLeader().getCoreName(); } - response.put(UNIQUE_KEY_FIELD_PARAM, schema.getUniqueKeyField().getName()); - - response.put(CONFIG_SET_PARAM, configSet); + response.uniqueKeyField = schema.getUniqueKeyField().getName(); + response.configSet = configSet; // important: pass the designer the current schema zk version for MVCC - response.put(SCHEMA_VERSION_PARAM, currentVersion); - response.put(TEMP_COLLECTION_PARAM, mutableId); - response.put("collectionsForConfig", configSetHelper.listCollectionsForConfig(configSet)); + response.schemaVersion = currentVersion; + response.tempCollection = mutableId; + response.collectionsForConfig = configSetHelper.listCollectionsForConfig(configSet); // Guess at a schema for each field found in the sample docs // Collect all fields across all docs with mapping to values - response.put( - "fields", + response.fields = schema.getFields().values().stream() .map(f -> fieldToMap(f, schema)) .sorted(Comparator.comparing(map -> ((String) map.get("name")))) - .collect(Collectors.toList())); + .collect(Collectors.toList()); if (settings == null) { settings = settingsDAO.getSettings(mutableId); } addSettingsToResponse(settings, response); - response.put( - "dynamicFields", + response.dynamicFields = Arrays.stream(schema.getDynamicFieldPrototypes()) .map(e -> e.getNamedPropertyValues(true)) .sorted(Comparator.comparing(map -> ((String) map.get("name")))) - .collect(Collectors.toList())); + .collect(Collectors.toList()); - response.put( - "fieldTypes", + response.fieldTypes = schema.getFieldTypes().values().stream() .map(fieldType -> fieldType.getNamedPropertyValues(true)) .sorted(Comparator.comparing(map -> ((String) map.get("name")))) - .collect(Collectors.toList())); + .collect(Collectors.toList()); // files SolrZkClient zkClient = zkStateReader().getZkClient(); @@ -1186,25 +1196,41 @@ protected Map buildResponse( List sortedFiles = new ArrayList<>(stripPrefix); Collections.sort(sortedFiles); - response.put("files", sortedFiles); + response.files = sortedFiles; // info about the sample docs if (docs != null) { final String uniqueKeyField = schema.getUniqueKeyField().getName(); - response.put( - "docIds", + response.docIds = docs.stream() .map(d -> (String) d.getFieldValue(uniqueKeyField)) .filter(Objects::nonNull) .limit(100) - .collect(Collectors.toList())); + .collect(Collectors.toList()); } - response.put("numDocs", docs != null ? docs.size() : -1); + response.numDocs = docs != null ? docs.size() : -1; return response; } + /** Sets the named schema-object field on {@code response} based on the action type. */ + private static void setSchemaObjectField( + SchemaDesignerResponse response, String action, Object value) { + // Handles both bare camelCase names used internally ('field', 'fieldType') and the + // kebab-case prefixed names that come directly from Schema API request JSON + // ('add-field', 'add-field-type', 'add-dynamic-field'). + switch (action) { + case "field", "add-field" -> response.field = value; + case "type", "add-type" -> response.type = value; + case "dynamicField", "add-dynamic-field" -> response.dynamicField = value; + case "fieldType", "add-field-type" -> response.fieldType = value; + default -> { + /* unknown action type — silently ignore */ + } + } + } + protected void addErrorToResponse( String collection, SolrException solrExc, @@ -1232,6 +1258,66 @@ protected void addErrorToResponse( } } + protected void addErrorToResponse( + String collection, + SolrException solrExc, + Map errorsDuringIndexing, + SchemaDesignerResponse response, + String updateError) { + + if (solrExc == null && (errorsDuringIndexing == null || errorsDuringIndexing.isEmpty())) { + return; // no errors + } + + if (updateError != null) { + response.updateError = updateError; + } + + if (solrExc != null) { + response.updateErrorCode = solrExc.code(); + if (response.updateError == null) { + response.updateError = solrExc.getMessage(); + } + } + + if (response.updateError == null) { + response.updateError = "Index sample documents into " + collection + " failed!"; + } + if (response.updateErrorCode == null) { + response.updateErrorCode = 400; + } + if (errorsDuringIndexing != null) { + response.errorDetails = errorsDuringIndexing; + } + } + + /** Overload for {@link SchemaDesignerPublishResponse} error fields. */ + protected void addErrorToResponse( + String collection, + SolrException solrExc, + Map errorsDuringIndexing, + SchemaDesignerPublishResponse response) { + + if (solrExc == null && (errorsDuringIndexing == null || errorsDuringIndexing.isEmpty())) { + return; // no errors + } + + if (solrExc != null) { + response.updateErrorCode = solrExc.code(); + response.updateError = solrExc.getMessage(); + } + + if (response.updateError == null) { + response.updateError = "Index sample documents into " + collection + " failed!"; + } + if (response.updateErrorCode == null) { + response.updateErrorCode = 400; + } + if (errorsDuringIndexing != null) { + response.errorDetails = errorsDuringIndexing; + } + } + protected SimpleOrderedMap fieldToMap(SchemaField f, ManagedIndexSchema schema) { SimpleOrderedMap map = f.getNamedPropertyValues(true); @@ -1246,8 +1332,8 @@ protected SimpleOrderedMap fieldToMap(SchemaField f, ManagedIndexSchema } @SuppressWarnings("unchecked") - protected Map readJsonFromRequest(SolrQueryRequest req) throws IOException { - ContentStream stream = extractSingleContentStream(req, true); + protected Map readJsonFromRequest() throws IOException { + ContentStream stream = extractSingleContentStream(true); String contentType = stream.getContentType(); if (StrUtils.isNullOrEmpty(contentType) || !contentType.toLowerCase(Locale.ROOT).contains(JSON_MIME)) { @@ -1260,8 +1346,7 @@ protected Map readJsonFromRequest(SolrQueryRequest req) throws I return (Map) json; } - protected void addSettingsToResponse( - SchemaDesignerSettings settings, final Map response) { + void addSettingsToResponse(SchemaDesignerSettings settings, final Map response) { response.put(LANGUAGES_PARAM, settings.getLanguages()); response.put(ENABLE_FIELD_GUESSING_PARAM, settings.fieldGuessingEnabled()); response.put(ENABLE_DYNAMIC_FIELDS_PARAM, settings.dynamicFieldsEnabled()); @@ -1275,7 +1360,40 @@ protected void addSettingsToResponse( } } - protected String checkMutable(String configSet, SolrQueryRequest req) throws IOException { + void addSettingsToResponse( + SchemaDesignerSettings settings, final SchemaDesignerInfoResponse response) { + response.languages = settings.getLanguages(); + response.enableFieldGuessing = settings.fieldGuessingEnabled(); + response.enableDynamicFields = settings.dynamicFieldsEnabled(); + response.enableNestedDocs = settings.nestedDocsEnabled(); + response.disabled = settings.isDisabled(); + settings.getPublishedVersion().ifPresent(v -> response.publishedVersion = v); + response.copyFrom = settings.getCopyFrom(); + } + + void addSettingsToResponse( + SchemaDesignerSettings settings, final SchemaDesignerSchemaDiffResponse response) { + response.languages = settings.getLanguages(); + response.enableFieldGuessing = settings.fieldGuessingEnabled(); + response.enableDynamicFields = settings.dynamicFieldsEnabled(); + response.enableNestedDocs = settings.nestedDocsEnabled(); + response.disabled = settings.isDisabled(); + settings.getPublishedVersion().ifPresent(v -> response.publishedVersion = v); + response.copyFrom = settings.getCopyFrom(); + } + + void addSettingsToResponse( + SchemaDesignerSettings settings, final SchemaDesignerResponse response) { + response.languages = settings.getLanguages(); + response.enableFieldGuessing = settings.fieldGuessingEnabled(); + response.enableDynamicFields = settings.dynamicFieldsEnabled(); + response.enableNestedDocs = settings.nestedDocsEnabled(); + response.disabled = settings.isDisabled(); + settings.getPublishedVersion().ifPresent(v -> response.publishedVersion = v); + response.copyFrom = settings.getCopyFrom(); + } + + protected String checkMutable(String configSet, int clientSchemaVersion) throws IOException { // an apply just copies over the temp config to the "live" location String mutableId = getMutableId(configSet); if (!configExists(mutableId)) { @@ -1290,34 +1408,27 @@ protected String checkMutable(String configSet, SolrQueryRequest req) throws IOE final int schemaVersionInZk = configSetHelper.getCurrentSchemaVersion(mutableId); if (schemaVersionInZk != -1) { // check the versions agree - configSetHelper.checkSchemaVersion( - mutableId, requireSchemaVersionFromClient(req), schemaVersionInZk); + configSetHelper.checkSchemaVersion(mutableId, clientSchemaVersion, schemaVersionInZk); } // else the stored is -1, can't really enforce here return mutableId; } - protected int requireSchemaVersionFromClient(SolrQueryRequest req) { - final int schemaVersion = req.getParams().getInt(SCHEMA_VERSION_PARAM, -1); - if (schemaVersion == -1) { + protected void requireSchemaVersion(Integer schemaVersion) { + if (schemaVersion == null || schemaVersion < 0) { throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - SCHEMA_VERSION_PARAM + " is a required parameter for the " + req.getPath() + " endpoint"); + SolrException.ErrorCode.BAD_REQUEST, SCHEMA_VERSION_PARAM + " is a required parameter!"); } - return schemaVersion; } - protected String getRequiredParam(final String param, final SolrQueryRequest req) { - final String paramValue = req.getParams().get(param); - if (StrUtils.isNullOrEmpty(paramValue)) { + protected void requireNotEmpty(final String param, final String value) { + if (StrUtils.isNullOrEmpty(value)) { throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - param + " is a required parameter for the " + req.getPath() + " endpoint!"); + SolrException.ErrorCode.BAD_REQUEST, param + " is a required parameter!"); } - return paramValue; } - protected void cleanupTemp(String configSet) throws IOException, SolrServerException { + protected void doCleanupTemp(String configSet) throws IOException, SolrServerException { String mutableId = getMutableId(configSet); indexedVersion.remove(mutableId); CollectionAdminRequest.deleteCollection(mutableId).process(cloudClient()); @@ -1325,6 +1436,13 @@ protected void cleanupTemp(String configSet) throws IOException, SolrServerExcep deleteConfig(mutableId); } + protected FlexibleSolrJerseyResponse buildFlexibleResponse(Map responseMap) { + FlexibleSolrJerseyResponse response = + instantiateJerseyResponse(FlexibleSolrJerseyResponse.class); + responseMap.forEach(response::setUnknownProperty); + return response; + } + private boolean configExists(String configSet) throws IOException { return coreContainer.getConfigSetService().checkConfigExists(configSet); } diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java index 23729eb21a14..ba76a8e1e51e 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java @@ -20,24 +20,18 @@ import static org.apache.solr.common.params.CommonParams.VERSION_FIELD; import static org.apache.solr.common.util.Utils.toJavabin; import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME; -import static org.apache.solr.handler.designer.SchemaDesignerAPI.getConfigSetZkPath; -import static org.apache.solr.handler.designer.SchemaDesignerAPI.getMutableId; +import static org.apache.solr.handler.designer.SchemaDesigner.getConfigSetZkPath; +import static org.apache.solr.handler.designer.SchemaDesigner.getMutableId; import static org.apache.solr.schema.IndexSchema.NEST_PATH_FIELD_NAME; import static org.apache.solr.schema.IndexSchema.ROOT_FIELD_NAME; import static org.apache.solr.schema.ManagedIndexSchemaFactory.DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME; -import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -47,16 +41,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.file.PathUtils; import org.apache.lucene.util.IOSupplier; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrResponse; @@ -74,7 +63,6 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkStateReader; @@ -383,7 +371,7 @@ boolean updateField( } } - // detect if they're trying to copy multi-valued fields into a single-valued field + // detect if they're trying to copy multivalued fields into a single-valued field Object multiValued = diff.get(MULTIVALUED); if (multiValued == null) { // mv not overridden explicitly, but we need the actual value, which will come from the new @@ -404,7 +392,7 @@ boolean updateField( name, src); multiValued = Boolean.TRUE; - diff.put(MULTIVALUED, multiValued); + diff.put(MULTIVALUED, true); break; } } @@ -415,8 +403,8 @@ boolean updateField( validateMultiValuedChange(configSet, schemaField, Boolean.FALSE); } - // switch from single-valued to multi-valued requires a full rebuild - // See SOLR-12185 ... if we're switching from single to multi-valued, then it's a big operation + // switch from single-valued to multivalued requires a full rebuild + // See SOLR-12185 ... if we're switching from single to multivalued, then it's a big operation if (fieldHasMultiValuedChange(multiValued, schemaField)) { needsRebuild = true; log.warn( @@ -536,29 +524,6 @@ static byte[] readAllBytes(IOSupplier hasStream) throws IOException } } - private String getBaseUrl(final String collection) { - String baseUrl = null; - try { - Set liveNodes = zkStateReader().getClusterState().getLiveNodes(); - DocCollection docColl = zkStateReader().getCollection(collection); - if (docColl != null && !liveNodes.isEmpty()) { - Optional maybeActive = - docColl.getReplicas().stream().filter(r -> r.isActive(liveNodes)).findAny(); - if (maybeActive.isPresent()) { - baseUrl = maybeActive.get().getBaseUrl(); - } - } - } catch (Exception exc) { - log.warn("Failed to lookup base URL for collection {}", collection, exc); - } - - if (baseUrl == null) { - baseUrl = zkStateReader().getBaseUrlForNodeName(cc.getZkController().getNodeName()); - } - - return baseUrl; - } - protected String getManagedSchemaZkPath(final String configSet) { return getConfigSetZkPath(configSet, DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME); } @@ -707,7 +672,7 @@ boolean applyCopyFieldUpdates( continue; // cannot copy to self } - // make sure the field exists and is multi-valued if this field is + // make sure the field exists and is multivalued if this field is SchemaField toAddField = schema.getFieldOrNull(toAdd); if (toAddField != null) { if (!field.multiValued() || toAddField.multiValued()) { @@ -817,8 +782,8 @@ protected ManagedIndexSchema removeLanguageSpecificObjectsAndFiles( final Set toRemove = types.values().stream() .filter(this::isTextType) - .filter(t -> !languages.contains(t.getTypeName().substring(TEXT_PREFIX_LEN))) .map(FieldType::getTypeName) + .filter(typeName -> !languages.contains(typeName.substring(TEXT_PREFIX_LEN))) .filter(t -> !usedTypes.contains(t)) // not explicitly used by a field .collect(Collectors.toSet()); @@ -959,9 +924,9 @@ protected ManagedIndexSchema restoreLanguageSpecificObjectsAndFiles( List addDynFields = Arrays.stream(copyFromSchema.getDynamicFields()) - .filter(df -> langFieldTypeNames.contains(df.getPrototype().getType().getTypeName())) - .filter(df -> !existingDynFields.contains(df.getPrototype().getName())) .map(IndexSchema.DynamicField::getPrototype) + .filter(prototype -> langFieldTypeNames.contains(prototype.getType().getTypeName())) + .filter(prototype -> !existingDynFields.contains(prototype.getName())) .collect(Collectors.toList()); if (!addDynFields.isEmpty()) { schema = schema.addDynamicFields(addDynFields, null, false); @@ -1033,8 +998,8 @@ protected ManagedIndexSchema restoreDynamicFields( .collect(Collectors.toSet()); List toAdd = Arrays.stream(dynamicFields) - .filter(df -> !existingDFNames.contains(df.getPrototype().getName())) .map(IndexSchema.DynamicField::getPrototype) + .filter(prototype -> !existingDFNames.contains(prototype.getName())) .collect(Collectors.toList()); // only restore language specific dynamic fields that match our langSet @@ -1098,52 +1063,6 @@ List listConfigsInZk() throws IOException { return cc.getConfigSetService().listConfigs(); } - byte[] downloadAndZipConfigSet(String configId) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Path tmpDirectory = - Files.createTempDirectory("schema-designer-" + FilenameUtils.getName(configId)); - try { - cc.getConfigSetService().downloadConfig(configId, tmpDirectory); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - tmpDirectory, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - - String dirName = tmpDirectory.relativize(dir).toString(); - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (!Files.isHidden(file)) { - try (InputStream fis = Files.newInputStream(file)) { - ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString()); - zipOut.putNextEntry(zipEntry); - fis.transferTo(zipOut); - } - } - return FileVisitResult.CONTINUE; - } - }); - } - } finally { - PathUtils.deleteDirectory(tmpDirectory); - } - return baos.toByteArray(); - } - protected ZkSolrResourceLoader zkLoaderForConfigSet(final String configSet) { SolrResourceLoader loader = cc.getResourceLoader(); return new ZkSolrResourceLoader( diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java index 0ad93d90d27c..5cb31ec954a6 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java @@ -21,18 +21,13 @@ public interface SchemaDesignerConstants { String CONFIG_SET_PARAM = "configSet"; String COPY_FROM_PARAM = "copyFrom"; String SCHEMA_VERSION_PARAM = "schemaVersion"; - String RELOAD_COLLECTIONS_PARAM = "reloadCollections"; - String INDEX_TO_COLLECTION_PARAM = "indexToCollection"; String NEW_COLLECTION_PARAM = "newCollection"; - String CLEANUP_TEMP_PARAM = "cleanupTemp"; String ENABLE_DYNAMIC_FIELDS_PARAM = "enableDynamicFields"; String ENABLE_FIELD_GUESSING_PARAM = "enableFieldGuessing"; String ENABLE_NESTED_DOCS_PARAM = "enableNestedDocs"; String TEMP_COLLECTION_PARAM = "tempCollection"; String PUBLISHED_VERSION = "publishedVersion"; - String DISABLE_DESIGNER_PARAM = "disableDesigner"; String DISABLED = "disabled"; - String DOC_ID_PARAM = "docId"; String FIELD_PARAM = "field"; String UNIQUE_KEY_FIELD_PARAM = "uniqueKeyField"; String AUTO_CREATE_FIELDS = "update.autoCreateFields"; diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettingsDAO.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettingsDAO.java index 9a09e8e5da94..939a89004978 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettingsDAO.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettingsDAO.java @@ -17,7 +17,7 @@ package org.apache.solr.handler.designer; -import static org.apache.solr.handler.designer.SchemaDesignerAPI.getConfigSetZkPath; +import static org.apache.solr.handler.designer.SchemaDesigner.getConfigSetZkPath; import java.io.IOException; import java.lang.invoke.MethodHandles; diff --git a/solr/core/src/java/org/apache/solr/handler/designer/package-info.java b/solr/core/src/java/org/apache/solr/handler/designer/package-info.java index 17e3b7af2761..c8516c1c3273 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/package-info.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/package-info.java @@ -20,5 +20,5 @@ * limitations under the License. */ -/** The {@link org.apache.solr.handler.designer.SchemaDesignerAPI} and supporting classes. */ +/** The {@link org.apache.solr.handler.designer.SchemaDesigner} and supporting classes. */ package org.apache.solr.handler.designer; diff --git a/solr/core/src/test/org/apache/solr/handler/configsets/DeleteConfigSetAPITest.java b/solr/core/src/test/org/apache/solr/handler/configsets/DeleteConfigSetAPITest.java new file mode 100644 index 000000000000..9846f960852e --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/configsets/DeleteConfigSetAPITest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.configsets; + +import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.mockito.Mockito.mock; + +import org.apache.solr.SolrTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Unit tests for {@link DeleteConfigSet}. + * + *

Note: This test focuses on input validation. Full deletion workflow is tested in integration + * tests like {@code TestConfigSetsAPI} since actual deletion requires ZooKeeper interaction. + */ +public class DeleteConfigSetAPITest extends SolrTestCase { + + private CoreContainer mockCoreContainer; + + @BeforeClass + public static void ensureWorkingMockito() { + assumeWorkingMockito(); + } + + @Before + public void clearMocks() { + mockCoreContainer = mock(CoreContainer.class); + } + + @Test + public void testNullConfigSetNameThrowsBadRequest() { + final var api = new DeleteConfigSet(mockCoreContainer, null, null); + final var ex = assertThrows(SolrException.class, () -> api.deleteConfigSet(null)); + + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue( + "Error message should mention missing configset name", + ex.getMessage().contains("No configset name")); + } + + @Test + public void testEmptyConfigSetNameThrowsBadRequest() { + final var api = new DeleteConfigSet(mockCoreContainer, null, null); + final var ex = assertThrows(SolrException.class, () -> api.deleteConfigSet("")); + + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue( + "Error message should mention missing configset name", + ex.getMessage().contains("No configset name")); + } + + @Test + public void testWhitespaceOnlyConfigSetNameThrowsBadRequest() { + final var api = new DeleteConfigSet(mockCoreContainer, null, null); + final var ex = assertThrows(SolrException.class, () -> api.deleteConfigSet(" ")); + + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue( + "Error message should mention missing configset name", + ex.getMessage().contains("No configset name")); + } + + @Test + public void testTabOnlyConfigSetNameThrowsBadRequest() { + final var api = new DeleteConfigSet(mockCoreContainer, null, null); + final var ex = assertThrows(SolrException.class, () -> api.deleteConfigSet("\t")); + + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue( + "Error message should mention missing configset name", + ex.getMessage().contains("No configset name")); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/configsets/DownloadConfigSetAPITest.java b/solr/core/src/test/org/apache/solr/handler/configsets/DownloadConfigSetAPITest.java new file mode 100644 index 000000000000..78984198271a --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/configsets/DownloadConfigSetAPITest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.configsets; + +import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import jakarta.ws.rs.core.Response; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.solr.SolrTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.FileSystemConfigSetService; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Unit tests for {@link DownloadConfigSet}. */ +public class DownloadConfigSetAPITest extends SolrTestCase { + + private CoreContainer mockCoreContainer; + private FileSystemConfigSetService configSetService; + private Path configSetBase; + + @BeforeClass + public static void ensureWorkingMockito() { + assumeWorkingMockito(); + } + + @Before + public void initConfigSetService() { + configSetBase = createTempDir("configsets"); + // Use an anonymous subclass to access the protected testing constructor + configSetService = new FileSystemConfigSetService(configSetBase) {}; + mockCoreContainer = mock(CoreContainer.class); + when(mockCoreContainer.getConfigSetService()).thenReturn(configSetService); + } + + /** Creates a configset directory with a single file so the API can find and zip it. */ + private void createConfigSet(String name, String fileName, String content) throws Exception { + Path dir = configSetBase.resolve(name); + Files.createDirectories(dir); + Files.writeString(dir.resolve(fileName), content, StandardCharsets.UTF_8); + } + + @Test + @SuppressWarnings("resource") // Response never created when exception is thrown + public void testMissingConfigSetNameThrowsBadRequest() { + final var api = new DownloadConfigSet(mockCoreContainer, null, null); + final var ex = assertThrows(SolrException.class, () -> api.downloadConfigSet(null, null)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + + final var ex2 = assertThrows(SolrException.class, () -> api.downloadConfigSet("", null)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex2.code()); + } + + @Test + @SuppressWarnings("resource") // Response never created when exception is thrown + public void testNonExistentConfigSetThrowsNotFound() { + // "missing" was never created in configSetBase, so checkConfigExists returns false + final var api = new DownloadConfigSet(mockCoreContainer, null, null); + final var ex = assertThrows(SolrException.class, () -> api.downloadConfigSet("missing", null)); + assertEquals(SolrException.ErrorCode.NOT_FOUND.code, ex.code()); + } + + @Test + public void testSuccessfulDownloadReturnsZipResponse() throws Exception { + createConfigSet("myconfig", "solrconfig.xml", ""); + + final var api = new DownloadConfigSet(mockCoreContainer, null, null); + try (final Response response = api.downloadConfigSet("myconfig", null)) { + assertNotNull(response); + assertEquals(200, response.getStatus()); + assertEquals("application/zip", response.getMediaType().toString()); + assertTrue( + String.valueOf(response.getHeaderString("Content-Disposition")) + .contains("myconfig_configset.zip")); + } + } + + @Test + public void testFilenameIsSanitized() throws Exception { + // A name with spaces gets sanitized: spaces → underscores in the Content-Disposition filename + final String nameWithSpaces = "my config name"; + createConfigSet(nameWithSpaces, "schema.xml", ""); + + final var api = new DownloadConfigSet(mockCoreContainer, null, null); + try (final Response response = api.downloadConfigSet(nameWithSpaces, null)) { + assertNotNull(response); + final String disposition = response.getHeaderString("Content-Disposition"); + assertTrue( + "filename must contain the sanitized (underscored) version of the name", + disposition.contains("my_config_name_configset.zip")); + assertFalse( + "filename must not retain spaces from the original configset name", + disposition.contains("my config name")); + } + } + + @Test + public void testDisplayNameOverridesFilename() throws Exception { + final String mutableId = "._designer_films"; + createConfigSet(mutableId, "schema.xml", ""); + + final var api = new DownloadConfigSet(mockCoreContainer, null, null); + try (final Response response = api.downloadConfigSet(mutableId, "films")) { + assertNotNull(response); + assertEquals(200, response.getStatus()); + final String disposition = response.getHeaderString("Content-Disposition"); + assertTrue( + "Content-Disposition should use the displayName 'films'", + disposition.contains("films_configset.zip")); + assertFalse( + "Content-Disposition must not expose the internal mutable-ID prefix", + disposition.contains("._designer_")); + } + } + + @Test + public void testBuildZipResponseUsesDisplayName() throws Exception { + createConfigSet("_designer_films", "schema.xml", ""); + + try (final Response response = + DownloadConfigSet.buildZipResponse(configSetService, "_designer_films", "films")) { + assertNotNull(response); + assertEquals(200, response.getStatus()); + final String disposition = response.getHeaderString("Content-Disposition"); + assertTrue( + "Content-Disposition should use the display name 'films'", + disposition.contains("films_configset.zip")); + assertFalse( + "Content-Disposition must not expose internal _designer_ prefix", + disposition.contains("_designer_")); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/configsets/GetConfigSetFileAPITest.java b/solr/core/src/test/org/apache/solr/handler/configsets/GetConfigSetFileAPITest.java new file mode 100644 index 000000000000..072924605ae7 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/configsets/GetConfigSetFileAPITest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.configsets; + +import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.solr.SolrTestCase; +import org.apache.solr.client.api.model.ConfigSetFileContentsResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.FileSystemConfigSetService; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Unit tests for {@link GetConfigSetFile}. */ +public class GetConfigSetFileAPITest extends SolrTestCase { + + private CoreContainer mockCoreContainer; + private FileSystemConfigSetService configSetService; + private Path configSetBase; + + @BeforeClass + public static void ensureWorkingMockito() { + assumeWorkingMockito(); + } + + @Before + public void initConfigSetService() { + configSetBase = createTempDir("configsets"); + // Use an anonymous subclass to access the protected testing constructor + configSetService = new FileSystemConfigSetService(configSetBase) {}; + mockCoreContainer = mock(CoreContainer.class); + when(mockCoreContainer.getConfigSetService()).thenReturn(configSetService); + } + + /** Creates a configset directory with one file. */ + private void createConfigSetWithFile(String configSetName, String filePath, String content) + throws Exception { + Path dir = configSetBase.resolve(configSetName); + Files.createDirectories(dir); + Files.writeString(dir.resolve(filePath), content, StandardCharsets.UTF_8); + } + + @Test + public void testMissingConfigSetNameThrowsBadRequest() { + final var api = new GetConfigSetFile(mockCoreContainer, null, null); + final var ex = + assertThrows(SolrException.class, () -> api.getConfigSetFile(null, "schema.xml")); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + + final var ex2 = assertThrows(SolrException.class, () -> api.getConfigSetFile("", "schema.xml")); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex2.code()); + } + + @Test + public void testMissingFilePathThrowsBadRequest() { + final var api = new GetConfigSetFile(mockCoreContainer, null, null); + final var ex = assertThrows(SolrException.class, () -> api.getConfigSetFile("myconfig", null)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + + final var ex2 = assertThrows(SolrException.class, () -> api.getConfigSetFile("myconfig", "")); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex2.code()); + } + + @Test + public void testNonExistentConfigSetThrowsNotFound() { + // "missing" was never created in configSetBase, so checkConfigExists returns false + final var api = new GetConfigSetFile(mockCoreContainer, null, null); + final var ex = + assertThrows(SolrException.class, () -> api.getConfigSetFile("missing", "schema.xml")); + assertEquals(SolrException.ErrorCode.NOT_FOUND.code, ex.code()); + } + + @Test + public void testSuccessfulFileRead() throws Exception { + final String configSetName = "myconfig"; + final String filePath = "schema.xml"; + final String fileContent = ""; + createConfigSetWithFile(configSetName, filePath, fileContent); + + final var api = new GetConfigSetFile(mockCoreContainer, null, null); + final ConfigSetFileContentsResponse response = api.getConfigSetFile(configSetName, filePath); + + assertNotNull(response); + assertEquals(filePath, response.path); + assertEquals(fileContent, response.content); + } + + @Test + public void testFileNotFoundInConfigSetThrowsNotFound() throws Exception { + final String configSetName = "myconfig"; + // Create the configset directory but do NOT add the requested file + Files.createDirectories(configSetBase.resolve(configSetName)); + + final var api = new GetConfigSetFile(mockCoreContainer, null, null); + final var ex = + assertThrows(SolrException.class, () -> api.getConfigSetFile(configSetName, "missing.xml")); + assertEquals(SolrException.ErrorCode.NOT_FOUND.code, ex.code()); + } + + @Test + public void testEmptyFileReturnsEmptyContent() throws Exception { + final String configSetName = "myconfig"; + final String filePath = "empty.xml"; + createConfigSetWithFile(configSetName, filePath, ""); + + final var api = new GetConfigSetFile(mockCoreContainer, null, null); + final ConfigSetFileContentsResponse response = api.getConfigSetFile(configSetName, filePath); + + assertNotNull(response); + assertEquals(filePath, response.path); + assertEquals("", response.content); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetAPITest.java b/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetAPITest.java new file mode 100644 index 000000000000..474b14b193ff --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetAPITest.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.configsets; + +import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.solr.SolrTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.FileSystemConfigSetService; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Unit tests for {@link UploadConfigSet}. */ +public class UploadConfigSetAPITest extends SolrTestCase { + + private CoreContainer mockCoreContainer; + private FileSystemConfigSetService configSetService; + private Path configSetBase; + + @BeforeClass + public static void ensureWorkingMockito() { + assumeWorkingMockito(); + } + + @Before + public void initConfigSetService() { + configSetBase = createTempDir("configsets"); + // Use an anonymous subclass to access the protected testing constructor + configSetService = new FileSystemConfigSetService(configSetBase) {}; + mockCoreContainer = mock(CoreContainer.class); + when(mockCoreContainer.getConfigSetService()).thenReturn(configSetService); + } + + /** Creates an in-memory ZIP file with the specified files. */ + @SuppressWarnings("try") // ZipOutputStream must be closed to finalize ZIP format + private InputStream createZipStream(String... filePathAndContent) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + for (int i = 0; i < filePathAndContent.length; i += 2) { + String filePath = filePathAndContent[i]; + String content = filePathAndContent[i + 1]; + zos.putNextEntry(new ZipEntry(filePath)); + zos.write(content.getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + } + } + return new ByteArrayInputStream(baos.toByteArray()); + } + + /** Creates an empty ZIP file. */ + @SuppressWarnings("try") // ZipOutputStream must be closed even with no entries + private InputStream createEmptyZipStream() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + // No entries + } + return new ByteArrayInputStream(baos.toByteArray()); + } + + /** Creates a configset with files on disk for testing overwrites and cleanup. */ + private void createExistingConfigSet(String configSetName, String... filePathAndContent) + throws Exception { + Path configDir = configSetBase.resolve(configSetName); + Files.createDirectories(configDir); + for (int i = 0; i < filePathAndContent.length; i += 2) { + String filePath = filePathAndContent[i]; + String content = filePathAndContent[i + 1]; + Path fullPath = configDir.resolve(filePath); + Files.createDirectories(fullPath.getParent()); + Files.writeString(fullPath, content, StandardCharsets.UTF_8); + } + } + + @Test + public void testSuccessfulZipUpload() throws Exception { + final String configSetName = "newconfig"; + InputStream zipStream = createZipStream("solrconfig.xml", ""); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + + assertNotNull(response); + assertTrue( + "ConfigSet should exist after upload", configSetService.checkConfigExists(configSetName)); + + // Verify the file was uploaded + byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, "solrconfig.xml"); + assertEquals("", new String(uploadedData, StandardCharsets.UTF_8)); + } + + @Test + public void testSuccessfulZipUploadWithMultipleFiles() throws Exception { + final String configSetName = "multifile"; + InputStream zipStream = + createZipStream( + "solrconfig.xml", "", + "schema.xml", "", + "stopwords.txt", "a\nthe"); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + + assertNotNull(response); + assertTrue(configSetService.checkConfigExists(configSetName)); + + // Verify all files were uploaded + byte[] solrconfig = configSetService.downloadFileFromConfig(configSetName, "solrconfig.xml"); + assertEquals("", new String(solrconfig, StandardCharsets.UTF_8)); + + byte[] schema = configSetService.downloadFileFromConfig(configSetName, "schema.xml"); + assertEquals("", new String(schema, StandardCharsets.UTF_8)); + + byte[] stopwords = configSetService.downloadFileFromConfig(configSetName, "stopwords.txt"); + assertEquals("a\nthe", new String(stopwords, StandardCharsets.UTF_8)); + } + + @Test + public void testEmptyZipThrowsBadRequest() throws Exception { + try (InputStream emptyZip = createEmptyZipStream()) { + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var ex = + assertThrows( + SolrException.class, () -> api.uploadConfigSet("newconfig", true, false, emptyZip)); + + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue( + "Error message should mention empty zip", + ex.getMessage().contains("empty zipped data") || ex.getMessage().contains("non-zipped")); + } + } + + @Test + public void testNonZipDataThrowsBadRequest() { + // Send plain text instead of a ZIP + InputStream notAZip = + new ByteArrayInputStream("this is not a zip file".getBytes(StandardCharsets.UTF_8)); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + // This should fail either as bad ZIP or as empty ZIP + assertThrows(Exception.class, () -> api.uploadConfigSet("newconfig", true, false, notAZip)); + } + + @Test + public void testOverwriteExistingConfigSet() throws Exception { + final String configSetName = "existing"; + // Create existing configset with old content + createExistingConfigSet(configSetName, "solrconfig.xml", ""); + + // Upload new content with overwrite=true + InputStream zipStream = createZipStream("solrconfig.xml", ""); + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + + assertNotNull(response); + + // Verify the file was overwritten + byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, "solrconfig.xml"); + assertEquals("", new String(uploadedData, StandardCharsets.UTF_8)); + } + + @Test + public void testOverwriteFalseThrowsExceptionWhenExists() throws Exception { + final String configSetName = "existing"; + createExistingConfigSet(configSetName, "solrconfig.xml", ""); + + try (InputStream zipStream = createZipStream("solrconfig.xml", "")) { + final var api = new UploadConfigSet(mockCoreContainer, null, null); + + final var ex = + assertThrows( + SolrException.class, () -> api.uploadConfigSet(configSetName, false, false, zipStream)); + + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue( + "Error message should mention config already exists", ex.getMessage().contains("already")); + } + } + + @Test + public void testCleanupRemovesUnusedFiles() throws Exception { + final String configSetName = "cleanuptest"; + // Create existing configset with multiple files + createExistingConfigSet( + configSetName, + "solrconfig.xml", + "", + "schema.xml", + "", + "old-file.txt", + "to be deleted"); + + // Upload new ZIP with only one file and cleanup=true + InputStream zipStream = createZipStream("solrconfig.xml", ""); + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSet(configSetName, true, true, zipStream); + + assertNotNull(response); + + // Verify solrconfig.xml was updated + byte[] solrconfig = configSetService.downloadFileFromConfig(configSetName, "solrconfig.xml"); + assertEquals("", new String(solrconfig, StandardCharsets.UTF_8)); + + // Verify old files were deleted (should throw or return null) + try { + byte[] oldSchema = configSetService.downloadFileFromConfig(configSetName, "schema.xml"); + if (oldSchema != null) { + fail("schema.xml should have been deleted during cleanup"); + } + } catch (Exception e) { + // Expected - file should not exist + } + } + + @Test + public void testCleanupFalseKeepsExistingFiles() throws Exception { + final String configSetName = "nocleanup"; + // Create existing configset with multiple files + createExistingConfigSet( + configSetName, "solrconfig.xml", "", "schema.xml", ""); + + // Upload new ZIP with only one file and cleanup=false + InputStream zipStream = createZipStream("solrconfig.xml", ""); + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + + assertNotNull(response); + + // Verify solrconfig.xml was updated + byte[] solrconfig = configSetService.downloadFileFromConfig(configSetName, "solrconfig.xml"); + assertEquals("", new String(solrconfig, StandardCharsets.UTF_8)); + + // Verify schema.xml still exists + byte[] schema = configSetService.downloadFileFromConfig(configSetName, "schema.xml"); + assertEquals("", new String(schema, StandardCharsets.UTF_8)); + } + + @Test + public void testSingleFileUploadSuccess() throws Exception { + final String configSetName = "singlefile"; + final String filePath = "solrconfig.xml"; + final String content = ""; + InputStream fileStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSetFile(configSetName, filePath, true, false, fileStream); + + assertNotNull(response); + + // Verify the file was uploaded + byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, filePath); + assertEquals(content, new String(uploadedData, StandardCharsets.UTF_8)); + } + + @Test + public void testSingleFileWithLeadingSlashIsNormalized() throws Exception { + final String configSetName = "leadingslash"; + final String filePath = "/solrconfig.xml"; // Leading slash + final String content = ""; + InputStream fileStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSetFile(configSetName, filePath, true, false, fileStream); + + assertNotNull(response); + + // Verify the file was uploaded without leading slash + byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, "solrconfig.xml"); + assertEquals(content, new String(uploadedData, StandardCharsets.UTF_8)); + } + + @Test + public void testSingleFileWithEmptyPathThrowsBadRequest() { + final String configSetName = "emptypath"; + InputStream fileStream = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + + // Test with empty string + final var ex = + assertThrows( + SolrException.class, + () -> api.uploadConfigSetFile(configSetName, "", true, false, fileStream)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue("Error should mention invalid path", ex.getMessage().contains("not valid")); + } + + @Test + public void testSingleFileWithNullPathThrowsBadRequest() { + final String configSetName = "nullpath"; + InputStream fileStream = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + + // Test with null - note: null becomes empty string after processing + final var ex = + assertThrows( + SolrException.class, + () -> api.uploadConfigSetFile(configSetName, null, true, false, fileStream)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + } + + @Test + public void testCleanupWithSingleFileThrowsBadRequest() { + final String configSetName = "nocleanupallowed"; + final String filePath = "solrconfig.xml"; + InputStream fileStream = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + + final var ex = + assertThrows( + SolrException.class, + () -> api.uploadConfigSetFile(configSetName, filePath, true, true, fileStream)); + + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue( + "Error should mention cleanup not allowed", ex.getMessage().contains("cleanup=true")); + } + + @Test + public void testDefaultParametersWhenNull() throws Exception { + final String configSetName = "defaults"; + InputStream zipStream = createZipStream("solrconfig.xml", ""); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + // Pass null for overwrite and cleanup - should use defaults (overwrite=true, cleanup=false) + final var response = api.uploadConfigSet(configSetName, null, null, zipStream); + + assertNotNull(response); + assertTrue(configSetService.checkConfigExists(configSetName)); + } + + @Test + public void testZipWithDirectoryEntries() throws Exception { + final String configSetName = "withdirs"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + // Add directory entry + zos.putNextEntry(new ZipEntry("conf/")); + zos.closeEntry(); + + // Add file in directory + zos.putNextEntry(new ZipEntry("conf/solrconfig.xml")); + zos.write("".getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + } + InputStream zipStream = new ByteArrayInputStream(baos.toByteArray()); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + + assertNotNull(response); + assertTrue(configSetService.checkConfigExists(configSetName)); + + // Directory entries should be skipped, but file should be uploaded + byte[] uploadedData = + configSetService.downloadFileFromConfig(configSetName, "conf/solrconfig.xml"); + assertEquals("", new String(uploadedData, StandardCharsets.UTF_8)); + } + + @Test + public void testOverwriteExistingFile() throws Exception { + final String configSetName = "overwritefile"; + final String filePath = "solrconfig.xml"; + + // Create existing file + createExistingConfigSet(configSetName, filePath, ""); + + // Upload new content with overwrite=true + InputStream fileStream = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSetFile(configSetName, filePath, true, false, fileStream); + + assertNotNull(response); + + // Verify file was overwritten + byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, filePath); + assertEquals("", new String(uploadedData, StandardCharsets.UTF_8)); + } + + @Test + public void testSingleFileUploadWithNestedPath() throws Exception { + final String configSetName = "nested"; + final String filePath = "lang/stopwords_en.txt"; + final String content = "a\nthe\nis"; + InputStream fileStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var response = api.uploadConfigSetFile(configSetName, filePath, true, false, fileStream); + + assertNotNull(response); + + // Verify the file was uploaded with correct path + byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, filePath); + assertEquals(content, new String(uploadedData, StandardCharsets.UTF_8)); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesigner.java similarity index 51% rename from solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java rename to solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesigner.java index 59dac14425a9..eedaf3d976b0 100644 --- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java +++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesigner.java @@ -19,11 +19,11 @@ import static org.apache.solr.common.params.CommonParams.JSON_MIME; import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME; -import static org.apache.solr.handler.designer.SchemaDesignerAPI.getMutableId; -import static org.apache.solr.response.RawResponseWriter.CONTENT; +import static org.apache.solr.handler.designer.SchemaDesigner.getMutableId; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -34,21 +34,27 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Stream; +import org.apache.solr.client.api.model.ConfigSetFileContentsResponse; +import org.apache.solr.client.api.model.FlexibleSolrJerseyResponse; +import org.apache.solr.client.api.model.SchemaDesignerCollectionsResponse; +import org.apache.solr.client.api.model.SchemaDesignerInfoResponse; +import org.apache.solr.client.api.model.SchemaDesignerResponse; +import org.apache.solr.client.api.model.SchemaDesignerSchemaDiffResponse; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.TestSampleDocumentsLoader; +import org.apache.solr.handler.configsets.GetConfigSetFile; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.ManagedIndexSchema; @@ -60,10 +66,11 @@ import org.junit.Test; import org.noggit.JSONUtil; -public class TestSchemaDesignerAPI extends SolrCloudTestCase implements SchemaDesignerConstants { +public class TestSchemaDesigner extends SolrCloudTestCase implements SchemaDesignerConstants { private CoreContainer cc; - private SchemaDesignerAPI schemaDesignerAPI; + private SchemaDesigner schemaDesigner; + private SolrQueryRequest mockReq; @BeforeClass public static void createCluster() throws Exception { @@ -87,39 +94,41 @@ public void setupTest() { assertNotNull(cluster); cc = cluster.getJettySolrRunner(0).getCoreContainer(); assertNotNull(cc); - schemaDesignerAPI = new SchemaDesignerAPI(cc); + mockReq = mock(SolrQueryRequest.class); + schemaDesigner = + new SchemaDesigner( + cc, + SchemaDesigner.newSchemaSuggester(), + SchemaDesigner.newSampleDocumentsLoader(), + mockReq); } public void testTSV() throws Exception { String configSet = "testTSV"; ModifiableSolrParams reqParams = new ModifiableSolrParams(); - - // GET /schema-designer/info - SolrQueryResponse rsp = new SolrQueryResponse(); - SolrQueryRequest req = mock(SolrQueryRequest.class); - reqParams.set(CONFIG_SET_PARAM, configSet); reqParams.set(LANGUAGES_PARAM, "en"); reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false); - when(req.getParams()).thenReturn(reqParams); + when(mockReq.getParams()).thenReturn(reqParams); String tsv = "id\tcol1\tcol2\n1\tfoo\tbar\n2\tbaz\tbah\n"; // POST some sample TSV docs ContentStream stream = new ContentStreamBase.StringStream(tsv, "text/csv"); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/analyze - schemaDesignerAPI.analyze(req, rsp); - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); - assertEquals(2, rsp.getValues().get("numDocs")); + SchemaDesignerResponse response = + schemaDesigner.analyze(configSet, null, null, null, List.of("en"), false, null, null); + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); + assertEquals(Integer.valueOf(2), response.numDocs); reqParams.clear(); reqParams.set(CONFIG_SET_PARAM, configSet); - rsp = new SolrQueryResponse(); - schemaDesignerAPI.cleanupTemp(req, rsp); + when(mockReq.getContentStreams()).thenReturn(null); + schemaDesigner.cleanupTempSchema(configSet); String mutableId = getMutableId(configSet); assertFalse(cc.getZkController().getClusterState().hasCollection(mutableId)); @@ -150,14 +159,8 @@ public void testAddTechproductsProgressively() throws Exception { String configSet = "techproducts"; - ModifiableSolrParams reqParams = new ModifiableSolrParams(); - // GET /schema-designer/info - SolrQueryResponse rsp = new SolrQueryResponse(); - SolrQueryRequest req = mock(SolrQueryRequest.class); - reqParams.set(CONFIG_SET_PARAM, configSet); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.getInfo(req, rsp); + SchemaDesignerInfoResponse infoResponse = schemaDesigner.getInfo(configSet); // response should just be the default values Map expSettings = Map.of( @@ -165,64 +168,46 @@ public void testAddTechproductsProgressively() throws Exception { ENABLE_FIELD_GUESSING_PARAM, true, ENABLE_NESTED_DOCS_PARAM, false, LANGUAGES_PARAM, List.of()); - assertDesignerSettings(expSettings, rsp.getValues()); - SolrParams rspData = rsp.getValues().toSolrParams(); - int schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + assertDesignerSettings(expSettings, infoResponse); + int schemaVersion = infoResponse.schemaVersion; assertEquals(schemaVersion, -1); // shouldn't exist yet // Use the prep endpoint to prepare the new schema - reqParams.clear(); - reqParams.set(CONFIG_SET_PARAM, configSet); - rsp = new SolrQueryResponse(); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.prepNewSchema(req, rsp); - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); - rspData = rsp.getValues().toSolrParams(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + SchemaDesignerResponse response = schemaDesigner.prepNewSchema(configSet, null); + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); + schemaVersion = response.schemaVersion; for (Path next : toAdd) { // Analyze some sample documents to refine the schema - reqParams.clear(); + ModifiableSolrParams reqParams = new ModifiableSolrParams(); reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set(LANGUAGES_PARAM, "en"); - reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); + when(mockReq.getParams()).thenReturn(reqParams); // POST some sample JSON docs ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(next); stream.setContentType( TestSampleDocumentsLoader.guessContentTypeFromFilename(next.getFileName().toString())); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/analyze - schemaDesignerAPI.analyze(req, rsp); + response = + schemaDesigner.analyze( + configSet, schemaVersion, null, null, List.of("en"), false, null, null); - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); - assertNotNull(rsp.getValues().get("fields")); - assertNotNull(rsp.getValues().get("fieldTypes")); - assertNotNull(rsp.getValues().get("docIds")); + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); + assertNotNull(response.fields); + assertNotNull(response.fieldTypes); + assertNotNull(response.docIds); // capture the schema version for MVCC - rspData = rsp.getValues().toSolrParams(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + schemaVersion = response.schemaVersion; } // get info (from the temp) - reqParams.clear(); - reqParams.set(CONFIG_SET_PARAM, configSet); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - rsp = new SolrQueryResponse(); - // GET /schema-designer/info - schemaDesignerAPI.getInfo(req, rsp); + infoResponse = schemaDesigner.getInfo(configSet); expSettings = Map.of( ENABLE_DYNAMIC_FIELDS_PARAM, false, @@ -230,57 +215,45 @@ public void testAddTechproductsProgressively() throws Exception { ENABLE_NESTED_DOCS_PARAM, false, LANGUAGES_PARAM, Collections.singletonList("en"), COPY_FROM_PARAM, "_default"); - assertDesignerSettings(expSettings, rsp.getValues()); + assertDesignerSettings(expSettings, infoResponse); // query to see how the schema decisions impact retrieval / ranking - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set(CommonParams.Q, "*:*"); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - rsp = new SolrQueryResponse(); + ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); + queryParams.set(CONFIG_SET_PARAM, configSet); + queryParams.set(CommonParams.Q, "*:*"); + when(mockReq.getParams()).thenReturn(queryParams); + when(mockReq.getContentStreams()).thenReturn(null); // GET /schema-designer/query - schemaDesignerAPI.query(req, rsp); - assertNotNull(rsp.getResponseHeader()); - SolrDocumentList results = (SolrDocumentList) rsp.getResponse(); - assertEquals(47, results.getNumFound()); + FlexibleSolrJerseyResponse queryResp = schemaDesigner.query(configSet); + assertNotNull(queryResp.unknownProperties().get("responseHeader")); + @SuppressWarnings("unchecked") + Map queryResponse = + (Map) queryResp.unknownProperties().get("response"); + assertNotNull("response object must be a map with numFound/docs", queryResponse); + assertEquals(47L, queryResponse.get("numFound")); + @SuppressWarnings("unchecked") + List queryDocs = (List) queryResponse.get("docs"); + assertNotNull("response.docs must be a list", queryDocs); + assertFalse("response.docs must be non-empty", queryDocs.isEmpty()); // publish schema to a config set that can be used by real collections - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - String collection = "techproducts"; - reqParams.set(NEW_COLLECTION_PARAM, collection); - reqParams.set(INDEX_TO_COLLECTION_PARAM, true); - reqParams.set(RELOAD_COLLECTIONS_PARAM, true); - reqParams.set(CLEANUP_TEMP_PARAM, true); - reqParams.set(DISABLE_DESIGNER_PARAM, true); - - rsp = new SolrQueryResponse(); - schemaDesignerAPI.publish(req, rsp); + schemaDesigner.publish(configSet, schemaVersion, collection, true, 1, 1, true, true, true); assertNotNull(cc.getZkController().zkStateReader.getCollection(collection)); // listCollectionsForConfig - reqParams.clear(); - reqParams.set(CONFIG_SET_PARAM, configSet); - rsp = new SolrQueryResponse(); - schemaDesignerAPI.listCollectionsForConfig(req, rsp); - List collections = (List) rsp.getValues().get("collections"); + SchemaDesignerCollectionsResponse collectionsResp = + schemaDesigner.listCollectionsForConfig(configSet); + List collections = collectionsResp.collections; assertNotNull(collections); assertTrue(collections.contains(collection)); // now try to create another temp, which should fail since designer is disabled for this // configSet now - reqParams.clear(); - reqParams.set(CONFIG_SET_PARAM, configSet); - rsp = new SolrQueryResponse(); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); try { - schemaDesignerAPI.prepNewSchema(req, rsp); + schemaDesigner.prepNewSchema(configSet, null); fail("Prep should fail for locked schema " + configSet); } catch (SolrException solrExc) { assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, solrExc.code()); @@ -292,38 +265,32 @@ public void testAddTechproductsProgressively() throws Exception { public void testSuggestFilmsXml() throws Exception { String configSet = "films"; - ModifiableSolrParams reqParams = new ModifiableSolrParams(); - Path filmsDir = ExternalPaths.SOURCE_HOME.resolve("example/films"); assertTrue(filmsDir + " not found!", Files.isDirectory(filmsDir)); Path filmsXml = filmsDir.resolve("films.xml"); assertTrue("example/films/films.xml not found", Files.isRegularFile(filmsXml)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, "true"); - - SolrQueryRequest req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - // POST some sample XML docs ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(filmsXml); stream.setContentType("application/xml"); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - - SolrQueryResponse rsp = new SolrQueryResponse(); + ModifiableSolrParams reqParams = new ModifiableSolrParams(); + reqParams.set(CONFIG_SET_PARAM, configSet); + when(mockReq.getParams()).thenReturn(reqParams); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/analyze - schemaDesignerAPI.analyze(req, rsp); - - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); - assertNotNull(rsp.getValues().get("fields")); - assertNotNull(rsp.getValues().get("fieldTypes")); - List docIds = (List) rsp.getValues().get("docIds"); + SchemaDesignerResponse response = + schemaDesigner.analyze(configSet, null, null, null, null, true, null, null); + + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); + assertNotNull(response.fields); + assertNotNull(response.fieldTypes); + List docIds = response.docIds; assertNotNull(docIds); assertEquals(100, docIds.size()); // designer limits the doc ids to top 100 - String idField = rsp.getValues()._getStr(UNIQUE_KEY_FIELD_PARAM); + String idField = response.uniqueKeyField; assertNotNull(idField); } @@ -332,16 +299,10 @@ public void testSuggestFilmsXml() throws Exception { public void testBasicUserWorkflow() throws Exception { String configSet = "testJson"; - ModifiableSolrParams reqParams = new ModifiableSolrParams(); - // Use the prep endpoint to prepare the new schema - reqParams.set(CONFIG_SET_PARAM, configSet); - SolrQueryResponse rsp = new SolrQueryResponse(); - SolrQueryRequest req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.prepNewSchema(req, rsp); - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); + SchemaDesignerResponse response = schemaDesigner.prepNewSchema(configSet, null); + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); Map expSettings = Map.of( @@ -350,44 +311,36 @@ public void testBasicUserWorkflow() throws Exception { ENABLE_NESTED_DOCS_PARAM, false, LANGUAGES_PARAM, List.of(), COPY_FROM_PARAM, "_default"); - assertDesignerSettings(expSettings, rsp.getValues()); + assertDesignerSettings(expSettings, response); // Analyze some sample documents to refine the schema - reqParams.clear(); - reqParams.set(CONFIG_SET_PARAM, configSet); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - - // POST some sample JSON docs Path booksJson = ExternalPaths.SOURCE_HOME.resolve("example/exampledocs/books.json"); ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(booksJson); stream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - - rsp = new SolrQueryResponse(); + ModifiableSolrParams reqParams = new ModifiableSolrParams(); + reqParams.set(CONFIG_SET_PARAM, configSet); + when(mockReq.getParams()).thenReturn(reqParams); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/analyze - schemaDesignerAPI.analyze(req, rsp); - - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); - assertNotNull(rsp.getValues().get("fields")); - assertNotNull(rsp.getValues().get("fieldTypes")); - assertNotNull(rsp.getValues().get("docIds")); - String idField = rsp.getValues()._getStr(UNIQUE_KEY_FIELD_PARAM); + response = schemaDesigner.analyze(configSet, null, null, null, null, null, null, null); + + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); + assertNotNull(response.fields); + assertNotNull(response.fieldTypes); + assertNotNull(response.docIds); + String idField = response.uniqueKeyField; assertNotNull(idField); - assertDesignerSettings(expSettings, rsp.getValues()); + assertDesignerSettings(expSettings, response); // capture the schema version for MVCC - SolrParams rspData = rsp.getValues().toSolrParams(); - reqParams.clear(); - int schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + int schemaVersion = response.schemaVersion; // load the contents of a file - Collection files = (Collection) rsp.getValues().get("files"); + Collection files = response.files; assertTrue(files != null && !files.isEmpty()); - reqParams.set(CONFIG_SET_PARAM, configSet); String file = null; for (String f : files) { if ("solrconfig.xml".equals(f)) { @@ -396,63 +349,37 @@ public void testBasicUserWorkflow() throws Exception { } } assertNotNull("solrconfig.xml not found in files!", file); - reqParams.set("file", file); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - rsp = new SolrQueryResponse(); - schemaDesignerAPI.getFileContents(req, rsp); - String solrconfigXml = (String) rsp.getValues().get(file); + GetConfigSetFile getFileApi = new GetConfigSetFile(cc, mockReq, mock(SolrQueryResponse.class)); + String fileMutableId = getMutableId(configSet); + ConfigSetFileContentsResponse fileContentsResp = + getFileApi.getConfigSetFile(fileMutableId, file); + String solrconfigXml = fileContentsResp.content; assertNotNull(solrconfigXml); - reqParams.clear(); // Update solrconfig.xml - rsp = new SolrQueryResponse(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set("file", file); - - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - when(req.getContentStreams()) + when(mockReq.getContentStreams()) .thenReturn( Collections.singletonList( new ContentStreamBase.StringStream(solrconfigXml, "application/xml"))); - - schemaDesignerAPI.updateFileContents(req, rsp); - rspData = rsp.getValues().toSolrParams(); - reqParams.clear(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + response = schemaDesigner.updateFileContents(configSet, file); + schemaVersion = response.schemaVersion; // update solrconfig.xml with some invalid XML mess - rsp = new SolrQueryResponse(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set("file", file); - - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - when(req.getContentStreams()) + when(mockReq.getContentStreams()) .thenReturn( Collections.singletonList( new ContentStreamBase.StringStream("", "application/xml"))); // this should fail b/c the updated solrconfig.xml is invalid - schemaDesignerAPI.updateFileContents(req, rsp); - rspData = rsp.getValues().toSolrParams(); - reqParams.clear(); - assertNotNull(rspData.get("updateFileError")); + response = schemaDesigner.updateFileContents(configSet, file); + assertNotNull(response.updateFileError); // remove dynamic fields and change the language to "en" only - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(null); // POST /schema-designer/analyze - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set(LANGUAGES_PARAM, "en"); - reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false); - reqParams.set(ENABLE_FIELD_GUESSING_PARAM, false); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.analyze(req, rsp); + response = + schemaDesigner.analyze( + configSet, schemaVersion, null, null, List.of("en"), false, false, null); expSettings = Map.of( @@ -461,28 +388,18 @@ public void testBasicUserWorkflow() throws Exception { ENABLE_NESTED_DOCS_PARAM, false, LANGUAGES_PARAM, Collections.singletonList("en"), COPY_FROM_PARAM, "_default"); - assertDesignerSettings(expSettings, rsp.getValues()); + assertDesignerSettings(expSettings, response); - List filesInResp = (List) rsp.getValues().get("files"); + List filesInResp = response.files; assertEquals(5, filesInResp.size()); assertTrue(filesInResp.contains("lang/stopwords_en.txt")); - rspData = rsp.getValues().toSolrParams(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); - - reqParams.clear(); + schemaVersion = response.schemaVersion; // add the dynamic fields back and change the languages too - rsp = new SolrQueryResponse(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.add(LANGUAGES_PARAM, "en"); - reqParams.add(LANGUAGES_PARAM, "fr"); - reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, true); - reqParams.set(ENABLE_FIELD_GUESSING_PARAM, false); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.analyze(req, rsp); + response = + schemaDesigner.analyze( + configSet, schemaVersion, null, null, Arrays.asList("en", "fr"), true, false, null); expSettings = Map.of( @@ -491,25 +408,18 @@ public void testBasicUserWorkflow() throws Exception { ENABLE_NESTED_DOCS_PARAM, false, LANGUAGES_PARAM, Arrays.asList("en", "fr"), COPY_FROM_PARAM, "_default"); - assertDesignerSettings(expSettings, rsp.getValues()); + assertDesignerSettings(expSettings, response); - filesInResp = (List) rsp.getValues().get("files"); + filesInResp = response.files; assertEquals(7, filesInResp.size()); assertTrue(filesInResp.contains("lang/stopwords_fr.txt")); - rspData = rsp.getValues().toSolrParams(); - reqParams.clear(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + schemaVersion = response.schemaVersion; - // add back all the default languages - rsp = new SolrQueryResponse(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.add(LANGUAGES_PARAM, "*"); - reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.analyze(req, rsp); + // add back all the default languages (using "*" wildcard -> empty list) + response = + schemaDesigner.analyze( + configSet, schemaVersion, null, null, List.of("*"), false, null, null); expSettings = Map.of( @@ -518,168 +428,109 @@ public void testBasicUserWorkflow() throws Exception { ENABLE_NESTED_DOCS_PARAM, false, LANGUAGES_PARAM, List.of(), COPY_FROM_PARAM, "_default"); - assertDesignerSettings(expSettings, rsp.getValues()); + assertDesignerSettings(expSettings, response); - filesInResp = (List) rsp.getValues().get("files"); + filesInResp = response.files; assertEquals(43, filesInResp.size()); assertTrue(filesInResp.contains("lang/stopwords_fr.txt")); assertTrue(filesInResp.contains("lang/stopwords_en.txt")); assertTrue(filesInResp.contains("lang/stopwords_it.txt")); - rspData = rsp.getValues().toSolrParams(); - reqParams.clear(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + schemaVersion = response.schemaVersion; // Get the value of a sample document String docId = "978-0641723445"; String fieldName = "series_t"; - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set(DOC_ID_PARAM, docId); - reqParams.set(FIELD_PARAM, fieldName); - reqParams.set(UNIQUE_KEY_FIELD_PARAM, idField); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - rsp = new SolrQueryResponse(); // GET /schema-designer/sample - schemaDesignerAPI.getSampleValue(req, rsp); - rspData = rsp.getValues().toSolrParams(); - assertNotNull(rspData.get(idField)); - assertNotNull(rspData.get(fieldName)); - assertNotNull(rspData.get("analysis")); - - reqParams.clear(); + FlexibleSolrJerseyResponse sampleResp = + schemaDesigner.getSampleValue(configSet, fieldName, idField, docId); + assertNotNull(sampleResp.unknownProperties().get(idField)); + assertNotNull(sampleResp.unknownProperties().get(fieldName)); + assertNotNull(sampleResp.unknownProperties().get("analysis")); // at this point the user would refine the schema by // editing suggestions for fields and adding/removing fields / field types as needed // add a new field - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); stream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-field.json")); stream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/add - schemaDesignerAPI.addSchemaObject(req, rsp); - assertNotNull(rsp.getValues().get("add-field")); - rspData = rsp.getValues().toSolrParams(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); - assertNotNull(rsp.getValues().get("fields")); + response = schemaDesigner.addSchemaObject(configSet, schemaVersion); + assertNotNull(response.field); + schemaVersion = response.schemaVersion; + assertNotNull(response.fields); // update an existing field - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - // switch a single-valued field to a multi-valued field, which triggers a full rebuild of the + // switch a single-valued field to a multivalued field, which triggers a full rebuild of the // "temp" collection stream = new ContentStreamBase.FileStream(getFile("schema-designer/update-author-field.json")); stream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // PUT /schema-designer/update - schemaDesignerAPI.updateSchemaObject(req, rsp); - assertNotNull(rsp.getValues().get("field")); - rspData = rsp.getValues().toSolrParams(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + response = schemaDesigner.updateSchemaObject(configSet, schemaVersion); + assertNotNull(response.field); + schemaVersion = response.schemaVersion; // add a new type - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); stream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-type.json")); stream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/add - schemaDesignerAPI.addSchemaObject(req, rsp); + response = schemaDesigner.addSchemaObject(configSet, schemaVersion); final String expectedTypeName = "test_txt"; - assertEquals(expectedTypeName, rsp.getValues().get("add-field-type")); - rspData = rsp.getValues().toSolrParams(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); - assertNotNull(rsp.getValues().get("fieldTypes")); - List> fieldTypes = - (List>) rsp.getValues().get("fieldTypes"); - Optional> expected = + assertEquals(expectedTypeName, response.fieldType); + schemaVersion = response.schemaVersion; + assertNotNull(response.fieldTypes); + @SuppressWarnings("unchecked") + List> fieldTypes = response.fieldTypes; + Optional> expected = fieldTypes.stream().filter(m -> expectedTypeName.equals(m.get("name"))).findFirst(); assertTrue( "New field type '" + expectedTypeName + "' not found in add type response!", expected.isPresent()); - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); stream = new ContentStreamBase.FileStream(getFile("schema-designer/update-type.json")); stream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/update - schemaDesignerAPI.updateSchemaObject(req, rsp); - rspData = rsp.getValues().toSolrParams(); - schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + response = schemaDesigner.updateSchemaObject(configSet, schemaVersion); + schemaVersion = response.schemaVersion; // query to see how the schema decisions impact retrieval / ranking - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set(CommonParams.Q, "*:*"); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - rsp = new SolrQueryResponse(); + ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); + queryParams.set(CONFIG_SET_PARAM, configSet); + queryParams.set(CommonParams.Q, "*:*"); + when(mockReq.getParams()).thenReturn(queryParams); + when(mockReq.getContentStreams()).thenReturn(null); // GET /schema-designer/query - schemaDesignerAPI.query(req, rsp); - assertNotNull(rsp.getResponseHeader()); - SolrDocumentList results = (SolrDocumentList) rsp.getResponse(); - assertEquals(4, results.size()); - - // Download ZIP - reqParams.clear(); - reqParams.set(CONFIG_SET_PARAM, configSet); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - rsp = new SolrQueryResponse(); - schemaDesignerAPI.downloadConfig(req, rsp); - assertNotNull(rsp.getValues().get(CONTENT)); + FlexibleSolrJerseyResponse queryResp2 = schemaDesigner.query(configSet); + assertNotNull(queryResp2.unknownProperties().get("responseHeader")); + @SuppressWarnings("unchecked") + Map queryResponse2 = + (Map) queryResp2.unknownProperties().get("response"); + assertNotNull("response object must be a map with numFound/docs", queryResponse2); + @SuppressWarnings("unchecked") + List queryDocs2 = (List) queryResponse2.get("docs"); + assertEquals(4, queryDocs2.size()); // publish schema to a config set that can be used by real collections - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - String collection = "test123"; - reqParams.set(NEW_COLLECTION_PARAM, collection); - reqParams.set(INDEX_TO_COLLECTION_PARAM, true); - reqParams.set(RELOAD_COLLECTIONS_PARAM, true); - reqParams.set(CLEANUP_TEMP_PARAM, true); - - rsp = new SolrQueryResponse(); - schemaDesignerAPI.publish(req, rsp); + schemaDesigner.publish(configSet, schemaVersion, collection, true, 1, 1, true, true, false); assertNotNull(cc.getZkController().zkStateReader.getCollection(collection)); // listCollectionsForConfig - reqParams.clear(); - reqParams.set(CONFIG_SET_PARAM, configSet); - rsp = new SolrQueryResponse(); - schemaDesignerAPI.listCollectionsForConfig(req, rsp); - List collections = (List) rsp.getValues().get("collections"); + SchemaDesignerCollectionsResponse collectionsResp2 = + schemaDesigner.listCollectionsForConfig(configSet); + List collections = collectionsResp2.collections; assertNotNull(collections); assertTrue(collections.contains(collection)); @@ -700,42 +551,28 @@ public void testBasicUserWorkflow() throws Exception { public void testFieldUpdates() throws Exception { String configSet = "fieldUpdates"; - ModifiableSolrParams reqParams = new ModifiableSolrParams(); - // Use the prep endpoint to prepare the new schema - reqParams.set(CONFIG_SET_PARAM, configSet); - SolrQueryResponse rsp = new SolrQueryResponse(); - SolrQueryRequest req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.prepNewSchema(req, rsp); - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); - SolrParams rspData = rsp.getValues().toSolrParams(); - int schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM); + SchemaDesignerResponse response = schemaDesigner.prepNewSchema(configSet, null); + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); + int schemaVersion = response.schemaVersion; // add our test field that we'll test various updates to - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion)); - reqParams.set(CONFIG_SET_PARAM, configSet); - req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-field.json")); stream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); // POST /schema-designer/add - schemaDesignerAPI.addSchemaObject(req, rsp); - assertNotNull(rsp.getValues().get("add-field")); + response = schemaDesigner.addSchemaObject(configSet, schemaVersion); + assertNotNull(response.field); final String fieldName = "keywords"; - Optional> maybeField = - ((List>) rsp.getValues().get("fields")) - .stream().filter(m -> fieldName.equals(m.get("name"))).findFirst(); + Optional> maybeField = + response.fields.stream().filter(m -> fieldName.equals(m.get("name"))).findFirst(); assertTrue(maybeField.isPresent()); - SimpleOrderedMap field = maybeField.get(); + Map field = maybeField.get(); assertEquals(Boolean.FALSE, field.get("indexed")); assertEquals(Boolean.FALSE, field.get("required")); assertEquals(Boolean.TRUE, field.get("stored")); @@ -746,22 +583,22 @@ public void testFieldUpdates() throws Exception { String mutableId = getMutableId(configSet); SchemaDesignerConfigSetHelper configSetHelper = - new SchemaDesignerConfigSetHelper(cc, SchemaDesignerAPI.newSchemaSuggester()); - ManagedIndexSchema schema = schemaDesignerAPI.loadLatestSchema(mutableId); + new SchemaDesignerConfigSetHelper(cc, SchemaDesigner.newSchemaSuggester()); + ManagedIndexSchema schema = schemaDesigner.loadLatestSchema(mutableId); // make it required Map updateField = Map.of("name", fieldName, "type", field.get("type"), "required", true); configSetHelper.updateField(configSet, updateField, schema); - schema = schemaDesignerAPI.loadLatestSchema(mutableId); + schema = schemaDesigner.loadLatestSchema(mutableId); SchemaField schemaField = schema.getField(fieldName); assertTrue(schemaField.isRequired()); updateField = Map.of("name", fieldName, "type", field.get("type"), "required", false, "stored", false); configSetHelper.updateField(configSet, updateField, schema); - schema = schemaDesignerAPI.loadLatestSchema(mutableId); + schema = schemaDesigner.loadLatestSchema(mutableId); schemaField = schema.getField(fieldName); assertFalse(schemaField.isRequired()); assertFalse(schemaField.stored()); @@ -779,7 +616,7 @@ public void testFieldUpdates() throws Exception { "multiValued", true); configSetHelper.updateField(configSet, updateField, schema); - schema = schemaDesignerAPI.loadLatestSchema(mutableId); + schema = schemaDesigner.loadLatestSchema(mutableId); schemaField = schema.getField(fieldName); assertFalse(schemaField.isRequired()); assertFalse(schemaField.stored()); @@ -787,7 +624,7 @@ public void testFieldUpdates() throws Exception { updateField = Map.of("name", fieldName, "type", "strings", "copyDest", "_text_"); configSetHelper.updateField(configSet, updateField, schema); - schema = schemaDesignerAPI.loadLatestSchema(mutableId); + schema = schemaDesigner.loadLatestSchema(mutableId); schemaField = schema.getField(fieldName); assertTrue(schemaField.multiValued()); assertEquals("strings", schemaField.getType().getTypeName()); @@ -801,44 +638,29 @@ public void testFieldUpdates() throws Exception { public void testSchemaDiffEndpoint() throws Exception { String configSet = "testDiff"; - ModifiableSolrParams reqParams = new ModifiableSolrParams(); - // Use the prep endpoint to prepare the new schema - reqParams.set(CONFIG_SET_PARAM, configSet); - SolrQueryResponse rsp = new SolrQueryResponse(); - SolrQueryRequest req = mock(SolrQueryRequest.class); - when(req.getParams()).thenReturn(reqParams); - schemaDesignerAPI.prepNewSchema(req, rsp); - assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM)); - assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM)); + SchemaDesignerResponse response = schemaDesigner.prepNewSchema(configSet, null); + assertNotNull(response.configSet); + assertNotNull(response.schemaVersion); + int schemaVersion = response.schemaVersion; // publish schema to a config set that can be used by real collections - reqParams.clear(); - reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(rsp.getValues().get(SCHEMA_VERSION_PARAM))); - reqParams.set(CONFIG_SET_PARAM, configSet); - String collection = "diff456"; - reqParams.set(NEW_COLLECTION_PARAM, collection); - reqParams.set(INDEX_TO_COLLECTION_PARAM, true); - reqParams.set(RELOAD_COLLECTIONS_PARAM, true); - reqParams.set(CLEANUP_TEMP_PARAM, true); - - rsp = new SolrQueryResponse(); - schemaDesignerAPI.publish(req, rsp); + schemaDesigner.publish(configSet, schemaVersion, collection, true, 1, 1, true, true, false); assertNotNull(cc.getZkController().zkStateReader.getCollection(collection)); // Load the schema designer for the existing config set and make some changes to it - reqParams.clear(); + ModifiableSolrParams reqParams = new ModifiableSolrParams(); reqParams.set(CONFIG_SET_PARAM, configSet); - reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, "true"); - reqParams.set(ENABLE_FIELD_GUESSING_PARAM, "false"); - rsp = new SolrQueryResponse(); - schemaDesignerAPI.analyze(req, rsp); + when(mockReq.getParams()).thenReturn(reqParams); + when(mockReq.getContentStreams()).thenReturn(null); + response = schemaDesigner.analyze(configSet, null, null, null, null, true, false, null); // Update id field to not use docValues + @SuppressWarnings("unchecked") List> fields = - (List>) rsp.getValues().get("fields"); + (List>) (List) response.fields; SimpleOrderedMap idFieldMap = fields.stream().filter(field -> field.get("name").equals("id")).findFirst().get(); idFieldMap.remove("copyDest"); // Don't include copyDest as it is not a property of SchemaField @@ -849,47 +671,39 @@ public void testSchemaDiffEndpoint() throws Exception { idFieldMapUpdated.setVal( idFieldMapUpdated.indexOf("omitTermFreqAndPositions", 0), Boolean.FALSE); - SolrParams solrParams = idFieldMapUpdated.toSolrParams(); - Map mapParams = solrParams.toMap(new HashMap<>()); + Map mapParams = idFieldMapUpdated.toSolrParams().toMap(new HashMap<>()); mapParams.put("termVectors", Boolean.FALSE); - reqParams.set( - SCHEMA_VERSION_PARAM, rsp.getValues().toSolrParams().getInt(SCHEMA_VERSION_PARAM)); + schemaVersion = response.schemaVersion; ContentStreamBase.StringStream stringStream = new ContentStreamBase.StringStream(JSONUtil.toJSON(mapParams), JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(stringStream)); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stringStream)); - rsp = new SolrQueryResponse(); - schemaDesignerAPI.updateSchemaObject(req, rsp); + response = schemaDesigner.updateSchemaObject(configSet, schemaVersion); // Add a new field - Integer schemaVersion = rsp.getValues().toSolrParams().getInt(SCHEMA_VERSION_PARAM); - reqParams.set(SCHEMA_VERSION_PARAM, schemaVersion); + schemaVersion = response.schemaVersion; ContentStreamBase.FileStream fileStream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-field.json")); fileStream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(fileStream)); - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(fileStream)); // POST /schema-designer/add - schemaDesignerAPI.addSchemaObject(req, rsp); - assertNotNull(rsp.getValues().get("add-field")); + response = schemaDesigner.addSchemaObject(configSet, schemaVersion); + assertNotNull(response.field); // Add a new field type - schemaVersion = rsp.getValues().toSolrParams().getInt(SCHEMA_VERSION_PARAM); - reqParams.set(SCHEMA_VERSION_PARAM, schemaVersion); + schemaVersion = response.schemaVersion; fileStream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-type.json")); fileStream.setContentType(JSON_MIME); - when(req.getContentStreams()).thenReturn(Collections.singletonList(fileStream)); - rsp = new SolrQueryResponse(); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(fileStream)); // POST /schema-designer/add - schemaDesignerAPI.addSchemaObject(req, rsp); - assertNotNull(rsp.getValues().get("add-field-type")); + response = schemaDesigner.addSchemaObject(configSet, schemaVersion); + assertNotNull(response.fieldType); // Let's do a diff now - rsp = new SolrQueryResponse(); - schemaDesignerAPI.getSchemaDiff(req, rsp); + SchemaDesignerSchemaDiffResponse diffResp = schemaDesigner.getSchemaDiff(configSet); - Map diff = (Map) rsp.getValues().get("diff"); + Map diff = diffResp.diff; // field asserts assertNotNull(diff.get("fields")); @@ -929,7 +743,76 @@ public void testSchemaDiffEndpoint() throws Exception { assertNotNull(fieldTypesAdded.get("test_txt")); } - protected void assertDesignerSettings(Map expected, NamedList actual) { + @Test + @SuppressWarnings("unchecked") + public void testQueryReturnsErrorDetailsOnIndexingFailure() throws Exception { + String configSet = "queryIndexErrTest"; + + // Prep the schema and analyze sample docs so the temp collection and stored docs exist + schemaDesigner.prepNewSchema(configSet, null); + ContentStreamBase.StringStream stream = + new ContentStreamBase.StringStream("[{\"id\":\"doc1\",\"title\":\"test doc\"}]", JSON_MIME); + ModifiableSolrParams reqParams = new ModifiableSolrParams(); + reqParams.set(CONFIG_SET_PARAM, configSet); + when(mockReq.getParams()).thenReturn(reqParams); + when(mockReq.getContentStreams()).thenReturn(Collections.singletonList(stream)); + schemaDesigner.analyze(configSet, null, null, null, null, null, null, null); + + // Build a fresh API instance whose indexedVersion cache is empty (so it always + // attempts to re-index before running the query), and which simulates indexing errors. + Map fakeErrors = new HashMap<>(); + fakeErrors.put("doc1", new RuntimeException("simulated indexing failure")); + SchemaDesigner apiWithErrors = + new SchemaDesigner( + cc, + SchemaDesigner.newSchemaSuggester(), + SchemaDesigner.newSampleDocumentsLoader(), + mockReq) { + @Override + protected Map indexSampleDocsWithRebuildOnAnalysisError( + String idField, + List docs, + String collectionName, + boolean asBatch, + String[] analysisErrorHolder) + throws IOException, SolrServerException { + return fakeErrors; + } + }; + + when(mockReq.getContentStreams()).thenReturn(null); + FlexibleSolrJerseyResponse response = apiWithErrors.query(configSet); + + Map props = response.unknownProperties(); + assertNotNull("updateError must be present in error response", props.get(UPDATE_ERROR)); + assertEquals(400, props.get("updateErrorCode")); + Map details = (Map) props.get(ERROR_DETAILS); + assertNotNull("errorDetails must be present in error response", details); + assertTrue("errorDetails must contain the failing doc id", details.containsKey("doc1")); + } + + @Test + public void testRequireSchemaVersionRejectsNegativeValues() throws Exception { + String configSet = "schemaVersionValidation"; + schemaDesigner.prepNewSchema(configSet, null); + + // null schemaVersion must be rejected + SolrException nullEx = + expectThrows(SolrException.class, () -> schemaDesigner.addSchemaObject(configSet, null)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, nullEx.code()); + + // negative schemaVersion must be rejected (was previously bypassing validation) + SolrException negEx = + expectThrows(SolrException.class, () -> schemaDesigner.addSchemaObject(configSet, -1)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, negEx.code()); + + // same contract must hold for updateSchemaObject + SolrException updateNegEx = + expectThrows(SolrException.class, () -> schemaDesigner.updateSchemaObject(configSet, -1)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, updateNegEx.code()); + } + + protected void assertDesignerSettings(Map expected, Map actual) { for (String expKey : expected.keySet()) { Object expValue = expected.get(expKey); assertEquals( @@ -938,4 +821,26 @@ protected void assertDesignerSettings(Map expected, NamedList actual.get(expKey)); } } + + protected void assertDesignerSettings( + Map expected, SchemaDesignerResponse response) { + Map actual = new HashMap<>(); + actual.put(LANGUAGES_PARAM, response.languages); + actual.put(ENABLE_FIELD_GUESSING_PARAM, response.enableFieldGuessing); + actual.put(ENABLE_DYNAMIC_FIELDS_PARAM, response.enableDynamicFields); + actual.put(ENABLE_NESTED_DOCS_PARAM, response.enableNestedDocs); + actual.put(COPY_FROM_PARAM, response.copyFrom); + assertDesignerSettings(expected, actual); + } + + protected void assertDesignerSettings( + Map expected, SchemaDesignerInfoResponse response) { + Map actual = new HashMap<>(); + actual.put(LANGUAGES_PARAM, response.languages); + actual.put(ENABLE_FIELD_GUESSING_PARAM, response.enableFieldGuessing); + actual.put(ENABLE_DYNAMIC_FIELDS_PARAM, response.enableDynamicFields); + actual.put(ENABLE_NESTED_DOCS_PARAM, response.enableNestedDocs); + actual.put(COPY_FROM_PARAM, response.copyFrom); + assertDesignerSettings(expected, actual); + } } diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java index 4552a8a65bc8..1a904a4e0720 100644 --- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java +++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java @@ -19,7 +19,7 @@ import static org.apache.solr.common.util.Utils.toJavabin; import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME; -import static org.apache.solr.handler.designer.SchemaDesignerAPI.getMutableId; +import static org.apache.solr.handler.designer.SchemaDesigner.getMutableId; import static org.apache.solr.schema.IndexSchema.NEST_PATH_FIELD_NAME; import static org.apache.solr.schema.IndexSchema.ROOT_FIELD_NAME; @@ -38,6 +38,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrConfig; import org.apache.solr.filestore.FileStore; +import org.apache.solr.handler.configsets.DownloadConfigSet; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.ManagedIndexSchema; import org.apache.solr.schema.SchemaField; @@ -75,7 +76,7 @@ public void setupTest() { assertNotNull(cluster); cc = cluster.getJettySolrRunner(0).getCoreContainer(); assertNotNull(cc); - helper = new SchemaDesignerConfigSetHelper(cc, SchemaDesignerAPI.newSchemaSuggester()); + helper = new SchemaDesignerConfigSetHelper(cc, SchemaDesigner.newSchemaSuggester()); } @Test @@ -113,13 +114,14 @@ public void testSetupMutable() throws Exception { configSet, schema, List.of(), true, DEFAULT_CONFIGSET_NAME); assertEquals(2, schema.getSchemaZkVersion()); - byte[] zipped = helper.downloadAndZipConfigSet(mutableId); + byte[] zipped = DownloadConfigSet.zipConfigSet(cc.getConfigSetService(), mutableId); assertTrue(zipped != null && zipped.length > 0); } @Test public void testDownloadAndZip() throws IOException { - byte[] zipped = helper.downloadAndZipConfigSet(DEFAULT_CONFIGSET_NAME); + byte[] zipped = + DownloadConfigSet.zipConfigSet(cc.getConfigSetService(), DEFAULT_CONFIGSET_NAME); ZipInputStream stream = new ZipInputStream(new ByteArrayInputStream(zipped)); boolean foundSolrConfig = false; @@ -177,7 +179,7 @@ public void testEnableDisableOptions() throws Exception { assertTrue( cluster .getZkClient() - .exists(SchemaDesignerAPI.getConfigSetZkPath(mutableId, "lang/stopwords_en.txt"))); + .exists(SchemaDesigner.getConfigSetZkPath(mutableId, "lang/stopwords_en.txt"))); assertNotNull(schema.getFieldTypeByName("text_fr")); assertNotNull(schema.getFieldOrNull("*_txt_fr")); assertNull(schema.getFieldOrNull("*_txt_ga")); @@ -200,7 +202,7 @@ public void testEnableDisableOptions() throws Exception { assertTrue( cluster .getZkClient() - .exists(SchemaDesignerAPI.getConfigSetZkPath(mutableId, "lang/stopwords_en.txt"))); + .exists(SchemaDesigner.getConfigSetZkPath(mutableId, "lang/stopwords_en.txt"))); assertNotNull(schema.getFieldTypeByName("text_fr")); assertNotNull(schema.getFieldOrNull("*_txt_fr")); assertNull(schema.getFieldOrNull("*_txt_ga")); diff --git a/solr/packaging/test/test_schema_designer.bats b/solr/packaging/test/test_schema_designer.bats new file mode 100644 index 000000000000..b769973c37b8 --- /dev/null +++ b/solr/packaging/test/test_schema_designer.bats @@ -0,0 +1,184 @@ +#!/usr/bin/env bats + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load bats_helper + +# A configSet name used throughout these tests +DESIGNER_CONFIGSET="bats_books" + +setup_file() { + common_clean_setup + solr start + solr assert --started http://localhost:${SOLR_PORT} --timeout 60000 +} + +teardown_file() { + common_setup + solr stop --all +} + +setup() { + common_setup +} + +teardown() { + save_home_on_failure + + # Best-effort cleanup of the designer draft so tests remain independent + curl -s -X DELETE "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}" > /dev/null || true +} + +# --------------------------------------------------------------------------- +# 1. List configs — the endpoint should return a JSON object with a configSets +# property even when no designer drafts exist yet. +# --------------------------------------------------------------------------- +@test "list schema-designer configs returns JSON with configSets key" { + run curl -s "http://localhost:${SOLR_PORT}/api/schema-designer/configs" + assert_output --partial '"configSets"' + refute_output --partial '"status":400' + refute_output --partial '"status":500' +} + +# --------------------------------------------------------------------------- +# 2. Prepare a new mutable draft configSet +# --------------------------------------------------------------------------- +@test "prep new schema-designer configSet" { + run curl -s -X POST \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/prep?copyFrom=_default" + assert_output --partial '"configSet"' + refute_output --partial '"status":400' + refute_output --partial '"status":500' +} + +# --------------------------------------------------------------------------- +# 3. Get info for the prepared configSet +# --------------------------------------------------------------------------- +@test "get info for schema-designer configSet" { + # Prepare first + curl -s -X POST \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/prep?copyFrom=_default" \ + > /dev/null + + run curl -s "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/info" + assert_output --partial '"configSet"' + assert_output --partial "${DESIGNER_CONFIGSET}" + refute_output --partial '"status":400' + refute_output --partial '"status":500' +} + +# --------------------------------------------------------------------------- +# 4. Analyze sample documents — sends books.json as the request body +# --------------------------------------------------------------------------- +@test "analyze sample documents for schema-designer configSet" { + # Prepare the draft first + curl -s -X POST \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/prep?copyFrom=_default" \ + > /dev/null + + run curl -s -X POST \ + -H "Content-Type: application/json" \ + --data-binary "@${SOLR_TIP}/example/exampledocs/books.json" \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/analyze" + assert_output --partial '"configSet"' + refute_output --partial '"status":400' + refute_output --partial '"status":500' +} + +# --------------------------------------------------------------------------- +# 5. Query the temporary collection — should return documents after analyze +# --------------------------------------------------------------------------- +@test "query schema-designer configSet returns documents" { + # Prepare and analyze to load sample docs + curl -s -X POST \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/prep?copyFrom=_default" \ + > /dev/null + curl -s -X POST \ + -H "Content-Type: application/json" \ + --data-binary "@${SOLR_TIP}/example/exampledocs/books.json" \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/analyze" \ + > /dev/null + + run curl -s \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/query?q=*:*" + assert_output --partial '"numFound"' + refute_output --partial '"status":400' + refute_output --partial '"status":500' +} + +# --------------------------------------------------------------------------- +# 6. Download configSet zip via the generic configsets endpoint. +# This is the primary new endpoint introduced by the migration. +# We verify: +# - HTTP 200 response +# - Content-Disposition header with a .zip filename +# - The response body is a valid zip (starts with the PK magic bytes) +# --------------------------------------------------------------------------- +@test "download schema-designer configSet as zip" { + # Prepare the draft + curl -s -X POST \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/prep?copyFrom=_default" \ + > /dev/null + + local mutable_id="._designer_${DESIGNER_CONFIGSET}" + local zip_file="${BATS_TEST_TMPDIR}/${DESIGNER_CONFIGSET}.zip" + + # Capture HTTP status code separately + local http_code + http_code=$(curl -s -o "${zip_file}" -w "%{http_code}" \ + "http://localhost:${SOLR_PORT}/api/configsets/${mutable_id}/download?displayName=${DESIGNER_CONFIGSET}") + + # Assert HTTP 200 + [ "${http_code}" = "200" ] + + # Assert the file was written and is non-empty + [ -s "${zip_file}" ] + + # Assert the file starts with the ZIP magic bytes (PK = 0x504B) + run bash -c "xxd '${zip_file}' | head -1" + assert_output --partial '504b' +} + +# --------------------------------------------------------------------------- +# 7. Download configSet zip — Content-Disposition header carries the filename +# --------------------------------------------------------------------------- +@test "download schema-designer configSet has correct Content-Disposition header" { + # Prepare the draft + curl -s -X POST \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/prep?copyFrom=_default" \ + > /dev/null + + local mutable_id="._designer_${DESIGNER_CONFIGSET}" + + run curl -s -I \ + "http://localhost:${SOLR_PORT}/api/configsets/${mutable_id}/download?displayName=${DESIGNER_CONFIGSET}" + assert_output --partial 'Content-Disposition' + assert_output --partial '.zip' +} + +# --------------------------------------------------------------------------- +# 8. Cleanup (DELETE) removes the designer draft +# --------------------------------------------------------------------------- +@test "cleanup schema-designer configSet succeeds" { + # Prepare first so there is something to delete + curl -s -X POST \ + "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}/prep?copyFrom=_default" \ + > /dev/null + + run curl -s -o /dev/null -w "%{http_code}" \ + -X DELETE "http://localhost:${SOLR_PORT}/api/schema-designer/${DESIGNER_CONFIGSET}" + assert_output "200" +} diff --git a/solr/webapp/web/js/angular/controllers/schema-designer.js b/solr/webapp/web/js/angular/controllers/schema-designer.js index 07013446a279..fdf7c68b7ba0 100644 --- a/solr/webapp/web/js/angular/controllers/schema-designer.js +++ b/solr/webapp/web/js/angular/controllers/schema-designer.js @@ -15,7 +15,7 @@ limitations under the License. */ -solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, $cookies, $window, Constants, SchemaDesigner, Luke) { +solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, $cookies, $window, Constants, SchemaDesigner, Configsets, Luke) { $scope.resetMenu("schema-designer", Constants.IS_ROOT_PAGE); $scope.schemas = []; @@ -887,10 +887,10 @@ solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, $scope.updateWorking = true; $scope.updateStatusMessage = "Updating file ..."; - SchemaDesigner.post(params, $scope.fileNodeText, function (data) { + SchemaDesigner.put(params, $scope.fileNodeText, function (data) { if (data.updateFileError) { - if (data[$scope.selectedFile]) { - $scope.fileNodeText = data[$scope.selectedFile]; + if (data.fileContent) { + $scope.fileNodeText = data.fileContent; } $scope.updateFileError = data.updateFileError; } else { @@ -904,9 +904,9 @@ solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, $scope.onSelectFileNode = function (id, doSelectOnTree) { $scope.selectedFile = id.startsWith("files/") ? id.substring("files/".length) : id; - var params = {path: "file", file: $scope.selectedFile, configSet: $scope.currentSchema}; - SchemaDesigner.get(params, function (data) { - $scope.fileNodeText = data[$scope.selectedFile]; + var mutableId = "._designer_" + $scope.currentSchema; + Configsets.get({configSetName: mutableId, endpoint: "file", path: $scope.selectedFile}, function (data) { + $scope.fileNodeText = data.content; $scope.isLeafNode = false; if (doSelectOnTree) { delete $scope.selectedNode; @@ -1523,10 +1523,12 @@ solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, $scope.downloadConfig = function () { // have to use an AJAX request so we can supply the Authorization header + var mutableId = "._designer_" + $scope.currentSchema; + var downloadUrl = "/api/configsets/" + encodeURIComponent(mutableId) + "/download?displayName=" + encodeURIComponent($scope.currentSchema); if (sessionStorage.getItem("auth.header")) { var fileName = $scope.currentSchema+"_configset.zip"; var xhr = new XMLHttpRequest(); - xhr.open("GET", "/api/schema-designer/download/"+fileName+"?wt=raw&configSet="+$scope.currentSchema, true); + xhr.open("GET", downloadUrl, true); xhr.setRequestHeader('Authorization', sessionStorage.getItem("auth.header")); xhr.responseType = 'blob'; xhr.addEventListener('load',function() { @@ -1543,7 +1545,7 @@ solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, }) xhr.send(); } else { - location.href = "/api/schema-designer/download/"+$scope.currentSchema+"_configset.zip?wt=raw&configSet=" + $scope.currentSchema; + location.href = downloadUrl; } }; @@ -1834,6 +1836,11 @@ solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, } SchemaDesigner.get(params, function (data) { + if (data.updateError != null) { + $scope.onError(data.updateError, data.updateErrorCode, data.errorDetails); + return; + } + $("#sort").trigger("chosen:updated"); $("#ff").trigger("chosen:updated"); $("#hl").trigger("chosen:updated"); diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js index 98e7e37d9baa..c72948546459 100644 --- a/solr/webapp/web/js/angular/services.js +++ b/solr/webapp/web/js/angular/services.js @@ -271,15 +271,22 @@ solrAdminServices.factory('System', }]) .factory('SchemaDesigner', ['$resource', function($resource) { - return $resource('/api/schema-designer/:path', {wt: 'json', path: '@path', _:Date.now()}, { + return $resource('/api/schema-designer/:configSet/:path', {wt: 'json', path: '@path', configSet: '@configSet', _:Date.now()}, { get: {method: "GET"}, post: {method: "POST", timeout: 90000}, put: {method: "PUT"}, + delete: {method: "DELETE"}, postXml: {headers: {'Content-type': 'text/xml'}, method: "POST", timeout: 90000}, postCsv: {headers: {'Content-type': 'application/csv'}, method: "POST", timeout: 90000}, upload: {method: "POST", transformRequest: angular.identity, headers: {'Content-Type': undefined}, timeout: 90000} }) }]) +.factory('Configsets', + ['$resource', function($resource) { + return $resource('/api/configsets/:configSetName/:endpoint', {wt: 'json', configSetName: '@configSetName', endpoint: '@endpoint', _:Date.now()}, { + get: {method: "GET"} + }) +}]) .factory('Security', ['$resource', function($resource) { return $resource('/api/cluster/security/:path', {wt: 'json', path: '@path', _:Date.now()}, {