Protobuf in Elixir

Protox is a great library to allow Protobuf to be used in Elixir.
While developing a span catcher for OpenTelemetry I found that I needed to decode a protobuf format message.

Here is the repo if you are interested:

https://github.com/chriseyre2000/span_eater

To construct the protobuf file I looked at the readme of opentelemetry_exporter, which pointed me to the protobuf definitions:

https://github.com/open-telemetry/opentelemetry-proto/tree/v0.11.0

I chose to simplify these into a single file (it’s not that large).

Protobuf is a wire format serializer. This means that if you send a Protobuf message any language that can use protobuf can read the message (provided that both side have the definition).

Given that this proto file has been implemented by a number of OpenTelemetry consumers it can be assumed to be stable.

Custom Tool to build protobuff files in Visual Studio

Here is the source to a visual studio addin for compiling protobuf files directly to cs source.
This is a great starting point for using an external compiler as a code generator that fits directly into visual studio.
This allows the definition of a Custom Tool that creates a code behind file that is automatically regenerated every time the top level item is saved.
It does require references to some items in the VS.NET SDK.
The error handling is fairly simplistic: Errors are embedded in place of the generated code.
The resulting dll needs to be registered for COM.  The registration code is as minimal as possible – even having the attribute use a constant.
using System;
using System.Globalization;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.VisualStudio.TextTemplating.VSHost;
using Microsoft.Win32;
using System.Diagnostics;
namespace ProtogenGenerator
{
[Guid(CustomGuidString)]
[ComVisible(true)]
public class MinimalTool : BaseCodeGeneratorWithSite
{
//This allows this to be defined once – the com registration routines need to match the declared guid
#warning Replace this with your own guid…
public const string CustomGuidString = “99C3D237-70D3-498c-BD54-CD108CC5E82A”;
private static Guid CustomToolGuid = new Guid( “{” + CustomGuidString + “}”);
private const string CustomToolName = “ProtogenGenerator”; // Need to update this…
private const string CustomToolDescription = “Generates ProtoGen”; // Need to update this…
private const string SourceFileExtension = “.proto”;
StringBuilder _errorBuffer = new StringBuilder();
StringBuilder _outputBuffer = new StringBuilder();
protected override byte[] GenerateCode(string inputFileName, string inputFileContent)
{
string rootName = Path.GetFileNameWithoutExtension(inputFileName);
string inputPath = Path.GetDirectoryName(inputFileName);
string toolFolder = Path.GetDirectoryName(typeof(MinimalTool).Assembly.Location);
ProcessStartInfo psi = new ProcessStartInfo( toolFolder + @”protoc.exe”);
psi.CreateNoWindow = true;
psi.WorkingDirectory = toolFolder;
psi.WindowStyle = ProcessWindowStyle.Hidden;
psi.RedirectStandardError = true;
psi.UseShellExecute = false;
StringBuilder arguments = new StringBuilder();
arguments.AppendFormat(@” –proto_path=””{0};{1}”””, toolFolder, inputPath);
arguments.AppendFormat(@” –descriptor_set_out=””{0}{1}.pb”””, toolFolder, rootName);
arguments.AppendFormat(@” “”{0}googleprotobuf{1}”””, toolFolder, “descriptor.proto”);
arguments.AppendFormat(@” “”{0}{1}”””, toolFolder, “csharp_options.proto”);
arguments.AppendFormat(@” “”{0}”””, inputFileName);
psi.Arguments = arguments.ToString();
int result;
using (Process proc = Process.Start(psi))
{
proc.EnableRaisingEvents = true;
proc.ErrorDataReceived += ProcErrorDataReceived;
proc.BeginErrorReadLine();
proc.WaitForExit();
result = proc.ExitCode;
proc.ErrorDataReceived -= ProcErrorDataReceived;
}
if (result != 0)
{
StringBuilder results = new StringBuilder();
results.Append(“There was a problem with the protoc compiler.”);
results.AppendLine();
results.Append(psi.FileName);
results.AppendLine();
results.Append(arguments.ToString());
results.AppendLine();
results.Append(_errorBuffer.ToString());
return Encoding.ASCII.GetBytes(results.ToString());
}
// Reset the string builder
_errorBuffer = new StringBuilder();
// We now have our “prebuild selector”
psi = new ProcessStartInfo(toolFolder + @”protogen.exe”);
psi.CreateNoWindow = true;
psi.WorkingDirectory = toolFolder;
psi.WindowStyle = ProcessWindowStyle.Hidden;
psi.RedirectStandardError = true;
psi.RedirectStandardOutput = true;
psi.UseShellExecute = false;
arguments = new StringBuilder();
string pbFileName = string.Format(@”{0}{1}.pb”, toolFolder, rootName);

 

arguments.AppendFormat(” ” + pbFileName);
psi.Arguments = arguments.ToString();
using (Process proc = Process.Start(psi))
{
proc.EnableRaisingEvents = true;
proc.ErrorDataReceived += ProcErrorDataReceived;
proc.OutputDataReceived += ProcOutputDataReceived;
proc.BeginOutputReadLine();
proc.BeginErrorReadLine();
proc.WaitForExit();
proc.ErrorDataReceived -= ProcErrorDataReceived;
proc.OutputDataReceived -= ProcOutputDataReceived;
result = proc.ExitCode;
}
if (result != 0)
{
StringBuilder results = new StringBuilder();
results.AppendFormat(“There was a problem with the protogen compiler {0}”, result);
results.AppendLine();
results.Append(psi.FileName);
results.AppendLine();
results.Append(arguments.ToString());
results.AppendLine();
results.AppendLine(“Error:”);
results.Append(_errorBuffer.ToString());
results.AppendLine();
results.AppendLine(“Output:”);
results.Append(_outputBuffer.ToString());
return Encoding.ASCII.GetBytes(results.ToString());
}
// I now need to find the name of the generated file
string sourceContent = File.ReadAllText(inputFileName);
string outputNamePrefix = null;
// I need to know the name of the generated cs file.
// Now I know the rules:
// 1. It can be specified in option
//     (google.protobuf.csharp_file_options).umbrella_classname = “PositionBuffer”;
// 2. If not specified there then:
//    take the file name, remove the extension
//    Convert to PascalCase, removing punctuation. Numbers and punctuation trigger new words.

Regex re = new Regex(“umbrella_classname[ ]*=[ ]*”(.*)”[ ]*;”);

if (re.IsMatch(sourceContent))
{
// We have pulled the umbrella_classname definition from the source.
outputNamePrefix = re.Match(sourceContent).Groups[1].Value;
}
else
{
outputNamePrefix = UnderscoresToPascalOrCamelCase(rootName, true);
}

if (outputNamePrefix == null)
{
return Encoding.ASCII.GetBytes(“Unable to determine the umbrella_classname”);
}
string outputFileName = toolFolder + “\” + outputNamePrefix + “Description.cs”;
if (!File.Exists(outputFileName))
{
// Try again
outputFileName = toolFolder + “\” + outputNamePrefix + “.cs”;
if (!File.Exists(outputFileName))
{
return Encoding.ASCII.GetBytes(“Unable to find file ” + outputFileName);
}
}
Byte[] resultBuffer = File.ReadAllBytes(outputFileName);
// This is the tidy up – only do this when the compile works.
File.Delete(pbFileName);
File.Delete(outputFileName);
return resultBuffer;
}
// This was taken from the FileDescriptor
private static string UnderscoresToPascalOrCamelCase(string input, bool pascal)
{
StringBuilder result = new StringBuilder();
bool capitaliseNext = pascal;
for (int i = 0; i < input.Length; i++)
{
char c = input[i];
if (‘a’ <= c && c <= ‘z’)
{
if (capitaliseNext)
{
result.Append(char.ToUpper(c, CultureInfo.InvariantCulture));
}
else
{
result.Append(c);
}
capitaliseNext = false;
}
else if (‘A’ <= c && c <= ‘Z’)
{
if (i == 0 && !pascal)
{
// Force first letter to lower-case unless explicitly told to capitalize it.
result.Append(char.ToLower(c, CultureInfo.InvariantCulture));
}
else
{
// Capital letters after the first are left as-is.
result.Append(c);
}
capitaliseNext = false;
}
else if (‘0’ <= c && c <= ‘9’)
{
result.Append(c);
capitaliseNext = true;
}
else
{
capitaliseNext = true;
}
}
return result.ToString();
}
void ProcOutputDataReceived(object sender, DataReceivedEventArgs e)
{
_outputBuffer.AppendLine(e.Data);
}
void ProcErrorDataReceived(object sender, DataReceivedEventArgs e)
{
_errorBuffer.AppendLine(e.Data);
}
public override string GetDefaultExtension()
{
return “.cs”;
}
#region Registration
private static Guid CSharpCategory =
new Guid(“{FAE04EC1-301F-11D3-BF4B-00C04F79EFBC}”);
private const string KeyFormat
= @”SOFTWAREMicrosoftVisualStudio{0}Generators{1}{2}”;
protected static void Register(Version vsVersion, Guid categoryGuid)
{
string subKey = String.Format(KeyFormat,
vsVersion, categoryGuid.ToString(“B”), CustomToolName);
using (RegistryKey key = Registry.LocalMachine.CreateSubKey(subKey))
{
key.SetValue(“”, CustomToolDescription);
key.SetValue(“CLSID”, CustomToolGuid.ToString(“B”));
key.SetValue(“GeneratesDesignTimeSource”, 1);
}
subKey = String.Format(KeyFormat,
vsVersion, categoryGuid.ToString(“B”), SourceFileExtension);
//This automates the association of the custom key with this tool.
using (RegistryKey key = Registry.LocalMachine.CreateSubKey(subKey))
{
key.SetValue(“”, CustomToolName);
}
}
protected static void Unregister(Version vsVersion, Guid categoryGuid)
{
string subKey = String.Format(KeyFormat,
vsVersion, categoryGuid.ToString(“B”), CustomToolName);
Registry.LocalMachine.DeleteSubKey(subKey, false);
subKey = String.Format(KeyFormat,
vsVersion, categoryGuid.ToString(“B”), SourceFileExtension);
Registry.LocalMachine.DeleteSubKey(subKey, false);
}
[ComRegisterFunction]
public static void RegisterClass(Type t)
{
// Register for both VS.NET 2002, 2003, 2008 and 2010  (C#)
Register(new Version(8, 0), CSharpCategory);
Register(new Version(9, 0), CSharpCategory);
Register(new Version(10, 0), CSharpCategory);
}
[ComUnregisterFunction]
public static void UnregisterClass(Type t)
{ // Unregister for both VS.NET 2002, 2003, 2008 and 2010 (C#)
Unregister(new Version(8, 0), CSharpCategory);
Unregister(new Version(9, 0), CSharpCategory);
Unregister(new Version(10, 0), CSharpCategory);
}
#endregion
}
}