1 package io.apicurio.registry.maven;
2
3 import com.fasterxml.jackson.databind.JsonNode;
4 import com.fasterxml.jackson.databind.ObjectMapper;
5 import com.google.protobuf.Descriptors.FileDescriptor;
6 import io.apicurio.registry.content.ContentHandle;
7 import io.apicurio.registry.content.TypedContent;
8 import io.apicurio.registry.content.refs.ExternalReference;
9 import io.apicurio.registry.content.refs.ReferenceFinder;
10 import io.apicurio.registry.maven.refs.IndexedResource;
11 import io.apicurio.registry.maven.refs.ReferenceIndex;
12 import io.apicurio.registry.rest.client.RegistryClient;
13 import io.apicurio.registry.rest.client.models.*;
14 import io.apicurio.registry.rules.ParsedJsonSchema;
15 import io.apicurio.registry.types.ArtifactType;
16 import io.apicurio.registry.types.ContentTypes;
17 import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider;
18 import io.apicurio.registry.types.provider.DefaultArtifactTypeUtilProviderImpl;
19 import io.vertx.core.Vertx;
20 import org.apache.avro.Schema;
21 import org.apache.commons.io.FileUtils;
22 import org.apache.maven.plugin.MojoExecutionException;
23 import org.apache.maven.plugin.MojoFailureException;
24 import org.apache.maven.plugins.annotations.Mojo;
25 import org.apache.maven.plugins.annotations.Parameter;
26
27 import java.io.*;
28 import java.nio.charset.StandardCharsets;
29 import java.nio.file.Files;
30 import java.nio.file.Path;
31 import java.nio.file.Paths;
32 import java.util.*;
33 import java.util.concurrent.ExecutionException;
34 import java.util.stream.Collectors;
35
36
37
38
39 @Mojo(name = "register")
40 public class RegisterRegistryMojo extends AbstractRegistryMojo {
41
42
43
44
45 @Parameter(required = false)
46 List<ExistingReference> existingReferences;
47
48
49
50
51 @Parameter(required = true)
52 List<RegisterArtifact> artifacts;
53
54
55
56
57 @Parameter(property = "skipRegister", defaultValue = "false")
58 boolean skip;
59
60
61
62
63
64 @Parameter(property = "dryRun", defaultValue = "false")
65 boolean dryRun;
66
67 private RegisterArtifact.AvroAutoRefsNamingStrategy avroAutoRefsNamingStrategy;
68
69 DefaultArtifactTypeUtilProviderImpl utilProviderFactory = new DefaultArtifactTypeUtilProviderImpl(true);
70
71
72
73
74 protected boolean validate() throws MojoExecutionException {
75 if (skip) {
76 getLog().info("register is skipped.");
77 return false;
78 }
79
80 if (artifacts == null || artifacts.isEmpty()) {
81 getLog().warn("No artifacts are configured for registration.");
82 return false;
83 }
84
85 if (existingReferences == null) {
86 existingReferences = new ArrayList<>();
87 }
88
89 int idx = 0;
90 int errorCount = 0;
91 for (RegisterArtifact artifact : artifacts) {
92 if (artifact.getGroupId() == null) {
93 getLog().error(String.format(
94 "GroupId is required when registering an artifact. Missing from artifacts[%d].",
95 idx));
96 errorCount++;
97 }
98 if (artifact.getArtifactId() == null) {
99 getLog().error(String.format(
100 "ArtifactId is required when registering an artifact. Missing from artifacts[%s].",
101 idx));
102 errorCount++;
103 }
104 if (artifact.getFile() == null) {
105 getLog().error(String.format(
106 "File is required when registering an artifact. Missing from artifacts[%s].", idx));
107 errorCount++;
108 } else if (!artifact.getFile().exists()) {
109 getLog().error(
110 String.format("Artifact file to register is configured but file does not exist: %s",
111 artifact.getFile().getPath()));
112 errorCount++;
113 }
114
115 idx++;
116 }
117
118 if (errorCount > 0) {
119 throw new MojoExecutionException(
120 "Invalid configuration of the Register Artifact(s) mojo. See the output log for details.");
121 }
122 return true;
123 }
124
125 @Override
126 protected void executeInternal() throws MojoExecutionException {
127 int errorCount = 0;
128 if (validate()) {
129 Vertx vertx = createVertx();
130 RegistryClient registryClient = createClient(vertx);
131
132 for (RegisterArtifact artifact : artifacts) {
133 String groupId = artifact.getGroupId();
134 String artifactId = artifact.getArtifactId();
135 try {
136 if (artifact.getAutoRefs() != null && artifact.getAutoRefs()) {
137
138
139 ReferenceIndex index = createIndex(artifact.getFile());
140 addExistingReferencesToIndex(registryClient, index, existingReferences);
141 addExistingReferencesToIndex(registryClient, index, artifact.getExistingReferences());
142 Stack<RegisterArtifact> registrationStack = new Stack<>();
143
144 this.avroAutoRefsNamingStrategy = artifact.getAvroAutoRefsNamingStrategy();
145 registerWithAutoRefs(registryClient, artifact, index, registrationStack);
146 } else if (artifact.getAnalyzeDirectory() != null && artifact.getAnalyzeDirectory()) {
147
148
149 getLog().warn("Deprecated feature in use: 'analyzeDirectory' -- use 'autoRefs' instead.");
150 registerDirectory(registryClient, artifact);
151 } else {
152
153 List<ArtifactReference> references = new ArrayList<>();
154
155 if (hasReferences(artifact)) {
156 references = processArtifactReferences(registryClient, artifact.getReferences());
157 }
158 registerArtifact(registryClient, artifact, references);
159 }
160 } catch (Exception e) {
161 errorCount++;
162 getLog().error(String.format("Exception while registering artifact [%s] / [%s]", groupId,
163 artifactId), e);
164 }
165
166 }
167
168 if (errorCount > 0) {
169 throw new MojoExecutionException("Errors while registering artifacts ...");
170 }
171 }
172 }
173
174 private VersionMetaData registerWithAutoRefs(RegistryClient registryClient, RegisterArtifact artifact,
175 ReferenceIndex index, Stack<RegisterArtifact> registrationStack) throws IOException,
176 ExecutionException, InterruptedException, MojoExecutionException, MojoFailureException {
177 if (loopDetected(artifact, registrationStack)) {
178 throw new RuntimeException(
179 "Artifact reference loop detected (not supported): " + printLoop(registrationStack));
180 }
181 registrationStack.push(artifact);
182
183
184 ContentHandle artifactContent = readContent(artifact.getFile());
185 String artifactContentType = getContentTypeByExtension(artifact.getFile().getName());
186 TypedContent typedArtifactContent = TypedContent.create(artifactContent, artifactContentType);
187
188
189 ArtifactTypeUtilProvider provider = this.utilProviderFactory
190 .getArtifactTypeProvider(artifact.getArtifactType());
191 ReferenceFinder referenceFinder = provider.getReferenceFinder();
192 var referenceArtifactIdentifierExtractor = provider.getReferenceArtifactIdentifierExtractor();
193 Set<ExternalReference> externalReferences = referenceFinder
194 .findExternalReferences(typedArtifactContent);
195
196
197 List<ArtifactReference> registeredReferences = new ArrayList<>(externalReferences.size());
198 for (ExternalReference externalRef : externalReferences) {
199 IndexedResource iresource = index.lookup(externalRef.getResource(),
200 Paths.get(artifact.getFile().toURI()));
201
202
203 if (iresource == null) {
204 throw new RuntimeException("Reference could not be resolved. From: "
205 + artifact.getFile().getName() + " To: " + externalRef.getFullReference());
206 }
207
208
209 if (!iresource.isRegistered()) {
210 String groupId = artifact.getGroupId();
211
212 String artifactId = referenceArtifactIdentifierExtractor.extractArtifactId(externalRef.getResource());
213 if(ArtifactType.AVRO.equals(iresource.getType())) {
214 if (avroAutoRefsNamingStrategy == RegisterArtifact.AvroAutoRefsNamingStrategy.USE_AVRO_NAMESPACE) {
215 groupId = referenceArtifactIdentifierExtractor.extractGroupId(externalRef.getResource());
216 artifactId = referenceArtifactIdentifierExtractor.extractArtifactId(externalRef.getResource());
217 }
218 if (avroAutoRefsNamingStrategy == RegisterArtifact.AvroAutoRefsNamingStrategy.INHERIT_PARENT_GROUP) {
219 groupId = artifact.getGroupId();
220 artifactId = iresource.getResourceName();
221 }
222 }
223 File localFile = getLocalFile(iresource.getPath());
224 RegisterArtifact refArtifact = buildFromRoot(artifact, artifactId, groupId);
225 refArtifact.setArtifactType(iresource.getType());
226 refArtifact.setVersion(null);
227 refArtifact.setFile(localFile);
228 refArtifact.setContentType(getContentTypeByExtension(localFile.getName()));
229 try {
230 var car = registerWithAutoRefs(registryClient, refArtifact, index, registrationStack);
231 iresource.setRegistration(car);
232 } catch (IOException | ExecutionException | InterruptedException e) {
233 throw new RuntimeException(e);
234 }
235 }
236
237 var reference = new ArtifactReference();
238 reference.setName(externalRef.getFullReference());
239 reference.setVersion(iresource.getRegistration().getVersion());
240 reference.setGroupId(iresource.getRegistration().getGroupId());
241 reference.setArtifactId(iresource.getRegistration().getArtifactId());
242 registeredReferences.add(reference);
243 }
244 registeredReferences.sort((ref1, ref2) -> ref1.getName().compareTo(ref2.getName()));
245
246 registrationStack.pop();
247 return registerArtifact(registryClient, artifact, registeredReferences);
248 }
249
250 private void registerDirectory(RegistryClient registryClient, RegisterArtifact artifact)
251 throws IOException, ExecutionException, InterruptedException, MojoExecutionException,
252 MojoFailureException {
253 switch (artifact.getArtifactType()) {
254 case ArtifactType.AVRO:
255 final AvroDirectoryParser avroDirectoryParser = new AvroDirectoryParser(registryClient);
256 final ParsedDirectoryWrapper<Schema> schema = avroDirectoryParser.parse(artifact.getFile());
257 registerArtifact(registryClient, artifact, avroDirectoryParser
258 .handleSchemaReferences(artifact, schema.getSchema(), schema.getSchemaContents()));
259 break;
260 case ArtifactType.PROTOBUF:
261 final ProtobufDirectoryParser protobufDirectoryParser = new ProtobufDirectoryParser(
262 registryClient);
263 final ParsedDirectoryWrapper<FileDescriptor> protoSchema = protobufDirectoryParser
264 .parse(artifact.getFile());
265 registerArtifact(registryClient, artifact, protobufDirectoryParser.handleSchemaReferences(
266 artifact, protoSchema.getSchema(), protoSchema.getSchemaContents()));
267 break;
268 case ArtifactType.JSON:
269 final JsonSchemaDirectoryParser jsonSchemaDirectoryParser = new JsonSchemaDirectoryParser(
270 registryClient);
271 final ParsedDirectoryWrapper<ParsedJsonSchema> jsonSchema = jsonSchemaDirectoryParser
272 .parse(artifact.getFile());
273 registerArtifact(registryClient, artifact, jsonSchemaDirectoryParser.handleSchemaReferences(
274 artifact, jsonSchema.getSchema(), jsonSchema.getSchemaContents()));
275 break;
276 default:
277 throw new IllegalArgumentException(
278 String.format("Artifact type not recognized for analyzing a directory structure %s",
279 artifact.getArtifactType()));
280 }
281 }
282
283 private VersionMetaData registerArtifact(RegistryClient registryClient, RegisterArtifact artifact,
284 List<ArtifactReference> references) throws FileNotFoundException, ExecutionException,
285 InterruptedException, MojoExecutionException, MojoFailureException {
286 if (artifact.getFile() != null) {
287 return registerArtifact(registryClient, artifact, new FileInputStream(artifact.getFile()),
288 references);
289 } else {
290 return getArtifactVersionMetadata(registryClient, artifact);
291 }
292 }
293
294 private VersionMetaData getArtifactVersionMetadata(RegistryClient registryClient,
295 RegisterArtifact artifact) {
296 String groupId = artifact.getGroupId();
297 String artifactId = artifact.getArtifactId();
298 String version = artifact.getVersion();
299
300 VersionMetaData amd = registryClient.groups().byGroupId(groupId).artifacts().byArtifactId(artifactId)
301 .versions().byVersionExpression(version).get();
302 getLog().info(String.format("Successfully processed artifact [%s] / [%s]. GlobalId is [%d]", groupId,
303 artifactId, amd.getGlobalId()));
304
305 return amd;
306 }
307
308 private VersionMetaData registerArtifact(RegistryClient registryClient, RegisterArtifact artifact,
309 InputStream artifactContent, List<ArtifactReference> references)
310 throws ExecutionException, InterruptedException, MojoFailureException, MojoExecutionException {
311 String groupId = artifact.getGroupId();
312 String artifactId = artifact.getArtifactId();
313 String version = artifact.getVersion();
314 String type = artifact.getArtifactType();
315 Boolean canonicalize = artifact.getCanonicalize();
316 Boolean isDraft = artifact.getIsDraft();
317 String ct = artifact.getContentType() == null ? ContentTypes.APPLICATION_JSON
318 : artifact.getContentType();
319 String data = null;
320 try {
321 if (artifact.getMinify() != null && artifact.getMinify()) {
322 ObjectMapper objectMapper = new ObjectMapper();
323 JsonNode jsonNode = objectMapper.readValue(artifactContent, JsonNode.class);
324 data = jsonNode.toString();
325 } else {
326 data = new String(artifactContent.readAllBytes(), StandardCharsets.UTF_8);
327 }
328 } catch (IOException e) {
329 throw new RuntimeException(e);
330 }
331
332 CreateArtifact createArtifact = new CreateArtifact();
333 createArtifact.setArtifactId(artifactId);
334 createArtifact.setArtifactType(type);
335
336 CreateVersion createVersion = new CreateVersion();
337 createVersion.setVersion(version);
338 createVersion.setIsDraft(isDraft);
339 createArtifact.setFirstVersion(createVersion);
340
341 VersionContent content = new VersionContent();
342 content.setContent(data);
343 content.setContentType(ct);
344 content.setReferences(references.stream().map(r -> {
345 ArtifactReference ref = new ArtifactReference();
346 ref.setArtifactId(r.getArtifactId());
347 ref.setGroupId(r.getGroupId());
348 ref.setVersion(r.getVersion());
349 ref.setName(r.getName());
350 return ref;
351 }).collect(Collectors.toList()));
352 createVersion.setContent(content);
353
354 try {
355 var vmd = registryClient.groups().byGroupId(groupId).artifacts().post(createArtifact, config -> {
356 if (artifact.getIfExists() != null) {
357 config.queryParameters.ifExists = IfArtifactExists
358 .forValue(artifact.getIfExists().value());
359 if (dryRun) {
360 config.queryParameters.dryRun = true;
361 }
362 }
363 config.queryParameters.canonical = canonicalize;
364 });
365
366 getLog().info(String.format("Successfully registered artifact [%s] / [%s]. GlobalId is [%d]",
367 groupId, artifactId, vmd.getVersion().getGlobalId()));
368
369
370 return vmd.getVersion();
371 } catch (RuleViolationProblemDetails | ProblemDetails e) {
372
373
374 if (Boolean.TRUE.equals(artifact.getIsDraft()) && e.getResponseStatusCode() == 409) {
375 try {
376 registryClient.groups().byGroupId(groupId).artifacts().byArtifactId(artifactId)
377 .versions().byVersionExpression(version).content()
378 .put(content, config -> {
379
380 });
381 getLog().info(String.format("Successfully updated artifact [%s] / [%s].",
382 groupId, artifactId));
383
384 return registryClient.groups().byGroupId(groupId).artifacts().byArtifactId(artifactId).versions().byVersionExpression(version).get();
385 } catch (RuleViolationProblemDetails | ProblemDetails pd) {
386 logAndThrow(pd);
387 return null;
388 }
389 } else {
390 logAndThrow(e);
391 return null;
392 }
393 }
394 }
395
396 private static boolean hasReferences(RegisterArtifact artifact) {
397 return artifact.getReferences() != null && !artifact.getReferences().isEmpty();
398 }
399
400 private List<ArtifactReference> processArtifactReferences(RegistryClient registryClient,
401 List<RegisterArtifactReference> referencedArtifacts) throws FileNotFoundException,
402 ExecutionException, InterruptedException, MojoExecutionException, MojoFailureException {
403 List<ArtifactReference> references = new ArrayList<>();
404 for (RegisterArtifactReference artifact : referencedArtifacts) {
405 List<ArtifactReference> nestedReferences = new ArrayList<>();
406
407
408 if (hasReferences(artifact)) {
409 nestedReferences = processArtifactReferences(registryClient, artifact.getReferences());
410 }
411 final VersionMetaData artifactMetaData = registerArtifact(registryClient, artifact,
412 nestedReferences);
413 references.add(buildReferenceFromMetadata(artifactMetaData, artifact.getName()));
414 }
415 return references;
416 }
417
418 public void setArtifacts(List<RegisterArtifact> artifacts) {
419 this.artifacts = artifacts;
420 }
421
422 public void setSkip(boolean skip) {
423 this.skip = skip;
424 }
425
426 private static ArtifactReference buildReferenceFromMetadata(VersionMetaData metaData,
427 String referenceName) {
428 ArtifactReference reference = new ArtifactReference();
429 reference.setName(referenceName);
430 reference.setArtifactId(metaData.getArtifactId());
431 reference.setGroupId(metaData.getGroupId());
432 reference.setVersion(metaData.getVersion());
433 return reference;
434 }
435
436 private static boolean isFileAllowedInIndex(File file) {
437 return file.isFile() && (
438 file.getName().toLowerCase().endsWith(".json") ||
439 file.getName().toLowerCase().endsWith(".yml") ||
440 file.getName().toLowerCase().endsWith(".yaml") ||
441 file.getName().toLowerCase().endsWith(".xml") ||
442 file.getName().toLowerCase().endsWith(".xsd") ||
443 file.getName().toLowerCase().endsWith(".wsdl") ||
444 file.getName().toLowerCase().endsWith(".graphql") ||
445 file.getName().toLowerCase().endsWith(".avsc") ||
446 file.getName().toLowerCase().endsWith(".proto")
447 );
448 }
449
450
451
452
453
454
455 private static ReferenceIndex createIndex(File file) {
456 ReferenceIndex index = new ReferenceIndex(file.getParentFile().toPath());
457 Collection<File> allFiles = FileUtils.listFiles(file.getParentFile(), null, true);
458 allFiles.stream().filter(RegisterRegistryMojo::isFileAllowedInIndex).forEach(f -> {
459 index.index(f.toPath(), readContent(f));
460 });
461 return index;
462 }
463
464 private void addExistingReferencesToIndex(RegistryClient registryClient, ReferenceIndex index,
465 List<ExistingReference> existingReferences) throws ExecutionException, InterruptedException {
466 if (existingReferences != null && !existingReferences.isEmpty()) {
467 for (ExistingReference ref : existingReferences) {
468 VersionMetaData vmd;
469 if (ref.getVersion() == null || "LATEST".equalsIgnoreCase(ref.getVersion())) {
470 vmd = registryClient.groups().byGroupId(ref.getGroupId()).artifacts()
471 .byArtifactId(ref.getArtifactId()).versions().byVersionExpression("branch=latest")
472 .get();
473 } else {
474 vmd = new VersionMetaData();
475 vmd.setGroupId(ref.getGroupId());
476 vmd.setArtifactId(ref.getArtifactId());
477 vmd.setVersion(ref.getVersion());
478 }
479 index.index(ref.getResourceName(), vmd);
480 }
481 }
482 }
483
484 protected static ContentHandle readContent(File file) {
485 try {
486 return ContentHandle.create(Files.readAllBytes(file.toPath()));
487 } catch (IOException e) {
488 throw new RuntimeException("Failed to read schema file: " + file, e);
489 }
490 }
491
492 protected static RegisterArtifact buildFromRoot(RegisterArtifact rootArtifact, String artifactId, String groupId) {
493 RegisterArtifact nestedSchema = new RegisterArtifact();
494 nestedSchema.setCanonicalize(rootArtifact.getCanonicalize());
495 nestedSchema.setArtifactId(artifactId);
496 nestedSchema.setGroupId(groupId == null ? rootArtifact.getGroupId() : groupId);
497 nestedSchema.setContentType(rootArtifact.getContentType());
498 nestedSchema.setArtifactType(rootArtifact.getArtifactType());
499 nestedSchema.setMinify(rootArtifact.getMinify());
500 nestedSchema.setContentType(rootArtifact.getContentType());
501 nestedSchema.setIfExists(rootArtifact.getIfExists());
502 nestedSchema.setAutoRefs(rootArtifact.getAutoRefs());
503 return nestedSchema;
504 }
505
506 private static File getLocalFile(Path path) {
507 return path.toFile();
508 }
509
510
511
512
513
514
515
516 private static boolean loopDetected(RegisterArtifact artifact,
517 Stack<RegisterArtifact> registrationStack) {
518 for (RegisterArtifact stackArtifact : registrationStack) {
519 if (artifact.getFile().equals(stackArtifact.getFile())) {
520 return true;
521 }
522 }
523 return false;
524 }
525
526 private static String printLoop(Stack<RegisterArtifact> registrationStack) {
527 return registrationStack.stream().map(artifact -> artifact.getFile().getName())
528 .collect(Collectors.joining(" -> "));
529 }
530
531 }