View Javadoc
1   package io.apicurio.registry.maven;
2   
3   import com.google.protobuf.Descriptors;
4   import com.squareup.wire.schema.internal.parser.ProtoFileElement;
5   import io.apicurio.registry.content.ContentHandle;
6   import io.apicurio.registry.content.TypedContent;
7   import io.apicurio.registry.rest.client.RegistryClient;
8   import io.apicurio.registry.rest.client.models.ArtifactReference;
9   import io.apicurio.registry.types.ContentTypes;
10  import io.apicurio.registry.utils.protobuf.schema.FileDescriptorUtils;
11  import org.slf4j.Logger;
12  import org.slf4j.LoggerFactory;
13  
14  import java.io.File;
15  import java.io.FileNotFoundException;
16  import java.util.ArrayList;
17  import java.util.Arrays;
18  import java.util.HashMap;
19  import java.util.HashSet;
20  import java.util.List;
21  import java.util.Map;
22  import java.util.Objects;
23  import java.util.Set;
24  import java.util.concurrent.ExecutionException;
25  import java.util.stream.Collectors;
26  
27  public class ProtobufDirectoryParser extends AbstractDirectoryParser<Descriptors.FileDescriptor> {
28  
29      private static final String PROTO_SCHEMA_EXTENSION = ".proto";
30      private static final Logger log = LoggerFactory.getLogger(ProtobufDirectoryParser.class);
31  
32      public ProtobufDirectoryParser(RegistryClient client) {
33          super(client);
34      }
35  
36      @Override
37      public ParsedDirectoryWrapper<Descriptors.FileDescriptor> parse(File protoFile) {
38  
39          Set<File> protoFiles = Arrays
40                  .stream(Objects.requireNonNull(protoFile.getParentFile()
41                          .listFiles((dir, name) -> name.endsWith(PROTO_SCHEMA_EXTENSION))))
42                  .filter(file -> !file.getName().equals(protoFile.getName())).collect(Collectors.toSet());
43  
44          try {
45              final Map<String, String> requiredSchemaDefs = new HashMap<>();
46              final Descriptors.FileDescriptor schemaDescriptor = FileDescriptorUtils
47                      .parseProtoFileWithDependencies(protoFile, protoFiles, requiredSchemaDefs);
48              assert allDependenciesHaveSamePackageName(requiredSchemaDefs, schemaDescriptor.getPackage())
49                      : "All dependencies must have the same package name as the main proto file";
50              Map<String, TypedContent> schemaContents = convertSchemaDefs(requiredSchemaDefs,
51                      schemaDescriptor.getPackage());
52              return new DescriptorWrapper(schemaDescriptor, schemaContents);
53          } catch (Descriptors.DescriptorValidationException e) {
54              throw new RuntimeException("Failed to read schema file: " + protoFile, e);
55          } catch (FileDescriptorUtils.ReadSchemaException e) {
56              log.warn(
57                      "Error processing Avro schema with name {}. This usually means that the references are not ready yet to read it",
58                      e.file());
59              throw new RuntimeException(e.getCause());
60          } catch (FileDescriptorUtils.ParseSchemaException e) {
61              log.warn(
62                      "Error processing Avro schema with name {}. This usually means that the references are not ready yet to parse it",
63                      e.fileName());
64              throw new RuntimeException(e.getCause());
65          }
66      }
67  
68      private static boolean allDependenciesHaveSamePackageName(Map<String, String> schemas,
69              String mainProtoPackageName) {
70          return schemas.keySet().stream().allMatch(fullDepName -> fullDepName.contains(mainProtoPackageName));
71      }
72  
73      /**
74       * Converts the schema definitions to a map of ContentHandle, stripping any package information from the
75       * key, which is not needed for the schema registry, given that the dependent schemas are *always* in the
76       * same package of the main proto file.
77       */
78      private Map<String, TypedContent> convertSchemaDefs(Map<String, String> requiredSchemaDefs,
79              String mainProtoPackageName) {
80          if (requiredSchemaDefs.isEmpty()) {
81              return Map.of();
82          }
83          Map<String, TypedContent> schemaDefs = new HashMap<>(requiredSchemaDefs.size());
84          for (Map.Entry<String, String> entry : requiredSchemaDefs.entrySet()) {
85              String fileName = FileDescriptorUtils.extractProtoFileName(entry.getKey());
86              TypedContent content = TypedContent.create(ContentHandle.create(entry.getValue()),
87                      ContentTypes.APPLICATION_PROTOBUF);
88              if (schemaDefs.put(fileName, content) != null) {
89                  log.warn(
90                          "There's a clash of dependency name, likely due to stripping the expected package name ie {}: dependencies: {}",
91                          mainProtoPackageName,
92                          Arrays.toString(requiredSchemaDefs.keySet().toArray(new Object[0])));
93              }
94          }
95          return schemaDefs;
96      }
97  
98      @Override
99      public List<ArtifactReference> handleSchemaReferences(RegisterArtifact rootArtifact,
100             Descriptors.FileDescriptor protoSchema, Map<String, TypedContent> fileContents)
101             throws FileNotFoundException, InterruptedException, ExecutionException {
102         Set<ArtifactReference> references = new HashSet<>();
103         final Set<Descriptors.FileDescriptor> baseDeps = new HashSet<>(
104                 Arrays.asList(FileDescriptorUtils.baseDependencies()));
105         final ProtoFileElement rootSchemaElement = FileDescriptorUtils
106                 .fileDescriptorToProtoFile(protoSchema.toProto());
107 
108         for (Descriptors.FileDescriptor dependency : protoSchema.getDependencies()) {
109 
110             List<ArtifactReference> nestedArtifactReferences = new ArrayList<>();
111             String dependencyFullName = dependency.getPackage() + "/" + dependency.getName(); // FIXME find a
112                                                                                               // better wat to
113                                                                                               // do this
114             if (!baseDeps.contains(dependency)
115                     && rootSchemaElement.getImports().contains(dependencyFullName)) {
116 
117                 RegisterArtifact nestedArtifact = buildFromRoot(rootArtifact, dependencyFullName);
118 
119                 if (!dependency.getDependencies().isEmpty()) {
120                     nestedArtifactReferences = handleSchemaReferences(nestedArtifact, dependency,
121                             fileContents);
122                 }
123 
124                 references.add(registerNestedSchema(dependencyFullName, nestedArtifactReferences,
125                         nestedArtifact, fileContents.get(dependency.getName()).getContent().content()));
126             }
127         }
128 
129         return new ArrayList<>(references);
130     }
131 
132     public static class DescriptorWrapper implements ParsedDirectoryWrapper<Descriptors.FileDescriptor> {
133         final Descriptors.FileDescriptor fileDescriptor;
134         final Map<String, TypedContent> schemaContents; // used to store the original file content to register
135                                                         // the content as-is.
136 
137         public DescriptorWrapper(Descriptors.FileDescriptor fileDescriptor,
138                 Map<String, TypedContent> schemaContents) {
139             this.fileDescriptor = fileDescriptor;
140             this.schemaContents = schemaContents;
141         }
142 
143         @Override
144         public Descriptors.FileDescriptor getSchema() {
145             return fileDescriptor;
146         }
147 
148         public Map<String, TypedContent> getSchemaContents() {
149             return schemaContents;
150         }
151     }
152 }