Skip to content

Commit

Permalink
Update convertToPdl tool to preserve source history
Browse files Browse the repository at this point in the history
RB=1904923
G=sf-reviewers
R=evwillia,kbalasub
A=evwillia
  • Loading branch information
nickibi committed Dec 11, 2019
1 parent 6fc9d37 commit ef20356
Show file tree
Hide file tree
Showing 5 changed files with 169 additions and 8 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
28.1.1
------
(RB=1904923)
Update convertToPdl tool to preserve source history

28.1.0
------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,7 @@ public class PegasusPlugin implements Plugin<Project>

private static final String CONVERT_TO_PDL_REVERSE = "convertToPdl.reverse";
private static final String CONVERT_TO_PDL_KEEP_ORIGINAL = "convertToPdl.keepOriginal";
private static final String CONVERT_TO_PDL_PRESERVE_SOURCE_CMD = "convertToPdl.preserveSourceCmd";

// Below variables are used to collect data across all pegasus projects (sub-projects) and then print information
// to the user at the end after build is finished.
Expand Down Expand Up @@ -1367,6 +1368,7 @@ protected void configureConversionUtilities(Project project, SourceSet sourceSet
File dataSchemaDir = project.file(getDataSchemaPath(project, sourceSet));
boolean reverse = isPropertyTrue(project, CONVERT_TO_PDL_REVERSE);
boolean keepOriginal = isPropertyTrue(project, CONVERT_TO_PDL_KEEP_ORIGINAL);
String preserveSourceCmd = getNonEmptyProperty(project, CONVERT_TO_PDL_PRESERVE_SOURCE_CMD);

// Utility task for migrating between PDSC and PDL.
project.getTasks().create(sourceSet.getTaskName("convert", "ToPdl"), TranslateSchemasTask.class, task ->
Expand All @@ -1375,6 +1377,7 @@ protected void configureConversionUtilities(Project project, SourceSet sourceSet
task.setDestinationDir(dataSchemaDir);
task.setResolverPath(getDataModelConfig(project, sourceSet));
task.setCodegenClasspath(project.getConfigurations().getByName("pegasusPlugin"));
task.setPreserveSourceCmd(preserveSourceCmd);
if (reverse)
{
task.setSourceFormat(SchemaFileType.PDL);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.OutputDirectory;
import org.gradle.api.tasks.PathSensitive;
import org.gradle.api.tasks.PathSensitivity;
Expand All @@ -28,6 +29,7 @@ public class TranslateSchemasTask extends DefaultTask {
private SchemaFileType _sourceFormat = SchemaFileType.PDSC;
private SchemaFileType _destinationFormat = SchemaFileType.PDL;
private boolean _keepOriginal = false;
private String _preserveSourceCmd;

@TaskAction
public void translate()
Expand All @@ -49,6 +51,11 @@ public void translate()
{
javaExecSpec.args("--keep-original");
}
if (_preserveSourceCmd != null)
{
javaExecSpec.args("--preserve-source");
javaExecSpec.args(_preserveSourceCmd);
}
javaExecSpec.args(resolverPathStr);
javaExecSpec.args(_inputDir.getAbsolutePath());
javaExecSpec.args(_destinationDir.getAbsolutePath());
Expand Down Expand Up @@ -142,4 +149,16 @@ public void setKeepOriginal(boolean keepOriginal)
{
_keepOriginal = keepOriginal;
}

@Input
@Optional
public String getPreserveSourceCmd()
{
return _preserveSourceCmd;
}

public void setPreserveSourceCmd(String preserveSourceCmd)
{
_preserveSourceCmd = preserveSourceCmd;
}
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
Copyright (c) 2019 LinkedIn Corp.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package com.linkedin.restli.tools.data;

import com.linkedin.data.schema.AbstractSchemaEncoder;
Expand Down Expand Up @@ -32,6 +48,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static com.linkedin.restli.tools.data.ScmUtil.DESTINATION;
import static com.linkedin.restli.tools.data.ScmUtil.SOURCE;

/**
* Command line tool to translate files between .pdl and .pdsc schema formats.
Expand All @@ -58,6 +76,10 @@ public class SchemaFormatTranslator
OPTIONS.addOption(OptionBuilder.withLongOpt("keep-original")
.withDescription("Keep the original files after translation (deleted by default)")
.create('o'));

OPTIONS.addOption(OptionBuilder.withLongOpt("preserve-source").hasArg()
.withDescription("Preserve source history command, use '" + SOURCE + "' as the source filename and use '" + DESTINATION + "' as the destination filename.")
.create('p'));
}

public static void main(String[] args) throws Exception
Expand All @@ -76,6 +98,7 @@ public static void main(String[] args) throws Exception
String sourceFormat = cl.getOptionValue('s', SchemaParser.FILETYPE).trim();
String destFormat = cl.getOptionValue('d', PdlSchemaParser.FILETYPE).trim();
boolean keepOriginal = cl.hasOption('o');
String preserveSourceCmd = cl.getOptionValue('p');

String[] cliArgs = cl.getArgs();
if (cliArgs.length != 3)
Expand Down Expand Up @@ -105,7 +128,7 @@ public static void main(String[] args) throws Exception
}

SchemaFormatTranslator translator =
new SchemaFormatTranslator(resolverPaths, sourceDir, destDir, sourceFormat, destFormat, keepOriginal);
new SchemaFormatTranslator(resolverPaths, sourceDir, destDir, sourceFormat, destFormat, keepOriginal, preserveSourceCmd);
translator.translateFiles();
}
catch (ParseException e)
Expand All @@ -122,26 +145,28 @@ public static void main(String[] args) throws Exception
private String _sourceFormat;
private String _destFormat;
private boolean _keepOriginal;
private String _preserveSourceCmd;

SchemaFormatTranslator(String resolverPath, File sourceDir, File destDir, String sourceFormat, String destFormat,
boolean keepOriginal)
boolean keepOriginal, String preserveSourceCmd)
{
_resolverPath = resolverPath;
_sourceDir = sourceDir;
_destDir = destDir;
_sourceFormat = sourceFormat;
_destFormat = destFormat;
_keepOriginal = keepOriginal;
_preserveSourceCmd = preserveSourceCmd;
}

private void translateFiles() throws IOException
private void translateFiles() throws IOException, InterruptedException
{
LOGGER.info("Translating files. Source dir: {}, sourceFormat: {}, destDir: {}, destFormat: {}, keepOriginal: {}",
_sourceDir, _sourceFormat, _destDir, _destFormat, _keepOriginal);
Map<String, SchemaInfo> topLevelSchemas = getTopLevelSchemaToTranslatedSchemaMap();
verifyTranslatedSchemas(topLevelSchemas);
// Write the destination files. Source files are deleted for this step unless keepOriginal flag is set.
writeTranslatedSchemasToDirectory(topLevelSchemas, _destDir, !_keepOriginal);
writeTranslatedSchemasToDirectory(topLevelSchemas, _destDir, !_keepOriginal, _preserveSourceCmd);
}

/**
Expand Down Expand Up @@ -172,13 +197,13 @@ private Map<String, SchemaInfo> getTopLevelSchemaToTranslatedSchemaMap() throws
return topLevelSchemas;
}

private void verifyTranslatedSchemas(Map<String, SchemaInfo> topLevelSchemas) throws IOException
private void verifyTranslatedSchemas(Map<String, SchemaInfo> topLevelSchemas) throws IOException, InterruptedException
{
File tempDir = new File(FileUtils.getTempDirectory(), "tmpPegasus");
FileUtils.deleteDirectory(tempDir);
assert tempDir.mkdirs();
// Write the schemas to temp directory for validation. Source files are not deleted/moved for this.
writeTranslatedSchemasToDirectory(topLevelSchemas, tempDir, false);
writeTranslatedSchemasToDirectory(topLevelSchemas, tempDir, false, null);

// Now try loading the schemas from the temp directory and compare with source schema.
StringTokenizer paths = new StringTokenizer(_resolverPath, File.pathSeparator);
Expand Down Expand Up @@ -228,7 +253,7 @@ private void verifyTranslatedSchemas(Map<String, SchemaInfo> topLevelSchemas) th
}

private void writeTranslatedSchemasToDirectory(
Map<String, SchemaInfo> topLevelSchemas, File outputDir, boolean moveSource) throws IOException
Map<String, SchemaInfo> topLevelSchemas, File outputDir, boolean moveSource, String preserveSourceCmd) throws IOException, InterruptedException
{
for (SchemaInfo schemaInfo : topLevelSchemas.values())
{
Expand All @@ -246,7 +271,7 @@ private void writeTranslatedSchemasToDirectory(
LOGGER.debug("Writing " + destinationFile.getAbsolutePath());
if (moveSource)
{
FileUtils.moveFile(schemaInfo.getSourceFile(), destinationFile);
ScmUtil.tryUpdateSourceHistory(preserveSourceCmd, schemaInfo.getSourceFile(), destinationFile);
}
FileUtils.writeStringToFile(destinationFile, schemaInfo.getDestEncodedSchemaString());
}
Expand Down
112 changes: 112 additions & 0 deletions restli-tools/src/main/java/com/linkedin/restli/tools/data/ScmUtil.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
Copyright (c) 2019 LinkedIn Corp.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package com.linkedin.restli.tools.data;


import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Basic Utilities for Source Code Management.
* @author ybi
*/
public class ScmUtil
{
private static final Logger LOGGER = LoggerFactory.getLogger(ScmUtil.class);

public static final String SOURCE = "$src";

public static final String DESTINATION = "$dst";

/**
* This method is used to preserve source history by running the given command.
*
* @param command preserve source history command which is passed by customer,
* it should contain $src as the name of sourceFile and $dst as the name of destinationFile.
* For example : "/usr/bin/svn mv \$src \$dst"
* @param sourceFile source file
* @param destinationFile destination file
* @throws IOException
* @throws InterruptedException
*/
public static void tryUpdateSourceHistory(String command, File sourceFile, File destinationFile) throws IOException, InterruptedException
{
if (isValidPreserveSourceCommand(command))
{
command = command.replace(SOURCE, sourceFile.getPath()).replace(DESTINATION, destinationFile.getPath());

StringBuilder stdout = new StringBuilder();
StringBuilder stderr = new StringBuilder();

if (executeWithStandardOutputAndError(command, stdout, stderr) != 0)
{
LOGGER.error("Could not run preserve source command : {} successfully. Please check the error message : {}", command, stderr.toString());
FileUtils.moveFile(sourceFile, destinationFile);
}
}
else
{
LOGGER.info("Preserve source command : {} is invalid.", command);
FileUtils.moveFile(sourceFile, destinationFile);
}
}

private static boolean isValidPreserveSourceCommand(String command)
{
return command != null && command.contains(SOURCE) && command.contains(DESTINATION);
}

private static int executeWithStandardOutputAndError(String command, StringBuilder stdout, StringBuilder stderr)
throws IOException, InterruptedException
{
Process process = execute(command);
stdout.append(getInputStreamAsString(process.getInputStream()));
stderr.append(getInputStreamAsString(process.getErrorStream()));
return process.exitValue();
}

private static Process execute(String command) throws IOException, InterruptedException
{
ProcessBuilder processBuilder = new ProcessBuilder(command.split("\\s+"));
Process process = processBuilder.start();
process.waitFor();
return process;
}

private static String getInputStreamAsString(InputStream input) throws IOException
{
StringBuilder result = new StringBuilder();
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, "UTF-8"));
boolean firstLine = true;
String line;
while ((line = bufferedReader.readLine()) != null)
{
if (!firstLine) {
result.append("\n");
}
firstLine = false;
result.append(line);
}
return result.toString();
}
}

0 comments on commit ef20356

Please sign in to comment.