Python: Parsing XML and Retaining the Comments

By default, Python’s built-in ElementTree module strips comments as it reads them. The solution is just obscure enough to be hard to find.

import xml.etree.ElementTree as ET

class _CommentedTreeBuilder(ET.TreeBuilder):
    def comment(self, data):
        self.start('!comment', {})
        self.data(data)
        self.end('!comment')

def parse(filepath):
    ctb = _CommentedTreeBuilder()
    xp = ET.XMLParser(target=ctb)
    tree = ET.parse(filepath, parser=xp)

    root = tree.getroot()
    # ...

When enumerating the parsed nodes, the comments will have a tag-name of “!comment”.

Advertisements

ssl: Promoting Existing Client Socket to SSL in C/C++

You may be in a situation where something else produces the sockets for you (such as an event-loop) or you otherwise need to manage the socket rather then allowing something else to.

#include <stdio.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <openssl/ssl.h>

int main(int argc, char *argv[])
{
    int sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (!sockfd) {
        printf("Error creating socket.\n");
        return -1;
    }

    struct sockaddr_in sa;
    memset (&sa, 0, sizeof(sa));

    sa.sin_family = AF_INET;
    sa.sin_addr.s_addr = inet_addr("172.217.2.196");
    sa.sin_port = htons (443); 

    socklen_t socklen = sizeof(sa);
    if (connect(sockfd, (struct sockaddr *)&sa, socklen)) {
        printf("Error connecting to server.\n");
        return -1;
    }

    SSL_library_init();
    SSLeay_add_ssl_algorithms();
    SSL_load_error_strings();

    const SSL_METHOD *meth = TLSv1_2_client_method();
    SSL_CTX *ctx = SSL_CTX_new (meth);

    SSL *ssl = SSL_new (ctx);
    if (ssl == NULL) {
        printf("Could not create SSL context.\n");
        return -1;
    }

    SSL_set_fd(ssl, sockfd);

    int err = SSL_connect(ssl);
    if (err <= 0) {
        printf("Could not connect.\n");
        return -1;
    }

    printf ("SSL connection using %s\n", SSL_get_cipher (ssl));

    // Do send/receive here.

    return 0;
}

Adapted from openssl-in-c-socket-connection-https-client, and works with both OpenSSL and BoringSSL.

Go: Generating KML

Use go-kml to build a KML structure. Then, marshal using the xml package.

An example using code from the project:

package main

import (
    "encoding/xml"
    "os"

    "github.com/twpayne/go-kml"
)

func main() {
    x := kml.KML(
        kml.Placemark(
            kml.Name("The Pentagon"),
            kml.Polygon(
                kml.Extrude(true),
                kml.AltitudeMode("relativeToGround"),
                kml.OuterBoundaryIs(
                    kml.LinearRing(
                        kml.Coordinates([]kml.Coordinate{
                            {-77.05788457660967, 38.87253259892824, 100},
                            {-77.05465973756702, 38.87291016281703, 100},
                            {-77.05315536854791, 38.87053267794386, 100},
                            {-77.05552622493516, 38.868757801256, 100},
                            {-77.05844056290393, 38.86996206506943, 100},
                            {-77.05788457660967, 38.87253259892824, 100},
                        }...),
                    ),
                ),
            ),
        ),
    )

    e := xml.NewEncoder(os.Stdout)
    e.Indent("", "  ")

    err := e.Encode(x)
    if err != nil {
        panic(err)
    }
}

This prints:

<kml xmlns="http://www.opengis.net/kml/2.2">
  <Placemark>
    <name>The Pentagon</name>
    <Polygon>
      <extrude>1</extrude>
      <altitudeMode>relativeToGround</altitudeMode>
      <outerBoundaryIs>
        <LinearRing>
          <coordinates>-77.05788457660967,38.87253259892824,100 -77.05465973756702,38.87291016281703,100 -77.0531553685479,38.87053267794386,100 -77.05552622493516,38.868757801256,100 -77.05844056290393,38.86996206506943,100 -77.05788457660967,38.87253259892824,100</coordinates>
        </LinearRing>
      </outerBoundaryIs>
    </Polygon>
  </Placemark>
</kml>

..which renders as this when using Google Earth:

Google Earth Pro_004

C++: Embedding the V8 JavaScript Engine

V8 is Chromium’s JavaScript interpreter. Not only can you use the included D8 utility to open a console and execute JavaScript code directly, but it is not as tough as you would think to integrate your native functions from your JavaScript and your JavaScript functions from your native functions. Of course, you have to get your head around all of the layers of isolates, context, and scoping.

For a walkthrough of embedding V8, start here.

In order to build a quick example of how to implement a native function for consumption from your JavaScript, go ahead and build V8. You will have to choose how to build the project. You’ll provide the name of a particular build configuration, which will write a few build parameters (called “args”) to the “out.gn//args.gn” file. This is a GN thing. GN is a build tool from the Chromium depot-tools project that generates Ninja scripts for doing the actual build.

For a list of build configurations, run:

$ tools/dev/v8gen.py list
...
s390.optdebug
s390.optdebug.sim
s390.release
s390.release.sim
s390x.debug
s390x.debug.sim
s390x.optdebug
s390x.optdebug.sim
s390x.release
s390x.release.sim
x64.debug
x64.optdebug
x64.release
x64.release.sample

(list shortened for simplicity)

We’ve used the “x64.release.sample” configuration for our example. It is straightforward for the purpose of this example. Once you have V8 downloaded and the dependencies installed, the actual project builds in about fifteen-minutes (with sixteen threads).

Drop this example into a file named “native_function.cpp” (or update the build command below to whatever you go with). This is an amalgam of excerpts from the samples in the project, examples in the documentation, and some customization on our part.

#include <sstream>

#include "libplatform/libplatform.h"
#include "v8.h"

// This is the function that we'll register into JS as a global function.
static void TestCallback(const v8::FunctionCallbackInfo<v8::Value>& args) {
  v8::Isolate* isolate = args.GetIsolate();
  v8::HandleScope scope(isolate);

  // Return [the arbitrary integer] 55 as our result.
  auto result = v8::Integer::New(isolate, 55);

  // Set the result.
  args.GetReturnValue().Set(result);
}

int main(int argc, char* argv[]) {

  // Initialize V8.
  v8::V8::InitializeICUDefaultLocation(argv[0]);
  v8::V8::InitializeExternalStartupData(argv[0]);
  std::unique_ptr<v8::Platform> platform = v8::platform::NewDefaultPlatform();
  v8::V8::InitializePlatform(platform.get());
  v8::V8::Initialize();

  // Create a new Isolate and make it the current one.
  v8::Isolate::CreateParams create_params;
  create_params.array_buffer_allocator =
      v8::ArrayBuffer::Allocator::NewDefaultAllocator();

  v8::Isolate* isolate = v8::Isolate::New(create_params);

  // Scoping in the C++ is tightly related to scoping in the JS. So, we'll 
  // retain the organizational blocks from the samples.
  {
    v8::Isolate::Scope isolate_scope(isolate);

    // Create a stack-allocated handle scope.
    v8::HandleScope handle_scope(isolate);

    // Create a global object to install our function into.
    v8::Local<v8::ObjectTemplate> global = v8::ObjectTemplate::New(isolate);

    // Register our global function.
    global->Set(
      v8::String::NewFromUtf8(isolate, "testCall", v8::NewStringType::kNormal).ToLocalChecked(),
      v8::FunctionTemplate::New(isolate, TestCallback)
    );

    // Create a new context and apply the global object.
    v8::Local<v8::Context> context = v8::Context::New(isolate, NULL, global);

    // Enter the context for compiling and running the hello world script.
    v8::Context::Scope context_scope(context);

    // Load the script.

    std::stringstream sourceStream;

    sourceStream
      << "testCall();" << std::endl;

    v8::Local<v8::String> source =
      v8::String::NewFromUtf8(
          isolate, 
          sourceStream.str().c_str(), 
          v8::NewStringType::kNormal)
        .ToLocalChecked();

    // Compile the source code.
    v8::Local<v8::Script> script =
      v8::Script::Compile(context, source)
        .ToLocalChecked();

    // Run the script.
    v8::Local<v8::Value> result = script->Run(context).ToLocalChecked();

    // Print the screen output.
    v8::String::Utf8Value output(isolate, result);
    printf("%s\n", *output);
  }

  // Dispose the isolate and tear down V8.
  isolate->Dispose();
  v8::V8::Dispose();
  v8::V8::ShutdownPlatform();
  delete create_params.array_buffer_allocator;

  return 0;
}

To build, set V8PATH to the absolute path of your “v8/v8” directory (the main V8 path established in the getting-started document above), and run the following:

$ g++ "-I${V8PATH}/include" -o native_function native_function.cpp -lv8_monolith "-L${V8PATH}/out.gn/x64.release.sample/obj" -pthread -std=c++0x

The example implements a simple, native function that just returns (55), registers it as a global JS function, creates a simple JS script that calls it, and then runs the script and prints the screen output. The screen output is just the result of that function printed to the screen since it was not otherwise assigned to a variable:

$ ./native_function 
55

For a couple of more examples, see the project repository.

Go: Read and Browse Ext4 Filesystems in User-Space

go-ext4 is a pure Go implementation of an Ext4 reader with support for reading the journal. An example of how to walk the file-structure:

inodeNumber := InodeRootDirectory

filepath := path.Join(assetsPath, "hierarchy_32.ext4")

f, err := os.Open(filepath)
log.PanicIf(err)

defer f.Close()

_, err = f.Seek(Superblock0Offset, io.SeekStart)
log.PanicIf(err)

sb, err := NewSuperblockWithReader(f)
log.PanicIf(err)

bgdl, err := NewBlockGroupDescriptorListWithReadSeeker(f, sb)
log.PanicIf(err)

bgd, err := bgdl.GetWithAbsoluteInode(inodeNumber)
log.PanicIf(err)

dw, err := NewDirectoryWalk(f, bgd, inodeNumber)
log.PanicIf(err)

allEntries := make([]string, 0)

for {
    fullPath, de, err := dw.Next()
    if err == io.EOF {
        break
    } else if err != nil {
        log.Panic(err)
    }

    description := fmt.Sprintf("%s: %s", fullPath, de.String())
    allEntries = append(allEntries, description)
}

sort.Strings(allEntries)

for _, entryDescription := range allEntries {
    fmt.Println(entryDescription)
}

// Output:
//
// directory1/fortune1: DirectoryEntry
// directory1/fortune2: DirectoryEntry
// directory1/fortune5: DirectoryEntry
// directory1/fortune6: DirectoryEntry
// directory1/subdirectory1/fortune3: DirectoryEntry
// directory1/subdirectory1/fortune4: DirectoryEntry
// directory1/subdirectory1: DirectoryEntry
// directory1/subdirectory2/fortune7: DirectoryEntry
// directory1/subdirectory2/fortune8: DirectoryEntry
// directory1/subdirectory2: DirectoryEntry
// directory1: DirectoryEntry
// directory2/fortune10: DirectoryEntry
// directory2/fortune9: DirectoryEntry
// directory2: DirectoryEntry
// lost+found: DirectoryEntry
// thejungle.txt: DirectoryEntry

This project is used to directly read the filesystem, file, and journal data without the support of kernel or the FUSE interface. Therefore, no elevated privileges are required.

Python: Retrieve User Info from LDAP

Supports returning the full DN for the user as well as a list of groups that they are a member of:

import logging
import ldap
import collections

# Install:
#
# apt: libsasl2-dev
# pip: python-ldap
#

_USER_QUERY = '(&(objectClass=USER)(sAMAccountName={username}))'
_GROUP_QUERY = '(&(objectClass=GROUP)(cn={group_name}))'

_LOGGER = logging.getLogger(__name__)

_USER = \
    collections.namedtuple(
        '_USER', [
            'dn',
            'attributes',
            'groups',
        ])


class NotFoundException(Exception):
    pass


class LdapAdapter(object):
    def __init__(self, host_and_port, username, password, base_dn):
        self.__host_and_port = host_and_port
        self.__username = username
        self.__password = password
        self.__base_dn = base_dn

        self.__raw_resource = None

    @property
    def __resource(self):
        if self.__raw_resource is None:
            self.__raw_resource = self.__auth()

        return self.__raw_resource

    def __auth(self):
        conn = ldap.initialize('ldap://' + self.__host_and_port)
        conn.protocol_version = 3
        conn.set_option(ldap.OPT_REFERRALS, 0)

        try:
            conn.simple_bind_s(self.__username, self.__password)
        except ldap.INVALID_CREDENTIALS:
            # Pinned, for the future.

            raise
        except ldap.SERVER_DOWN:
            # Pinned, for the future.

            raise
        except ldap.LDAPError:
            _LOGGER.error("LDAP error content:\n{}".format(e.message))

            if issubclass(e.message.__class__, dict) is True and \
               'desc' in e.message:
                raise Exception("LDAP: {}".format(e.message['desc']))

            raise
        else:
            return conn

    def get_dn_by_username(self, username):
        """Return user information. See _USER."""

        uf = _USER_QUERY.format(username=username)
        results_raw = \
            self.__resource.search_s(
                self.__base_dn,
                ldap.SCOPE_SUBTREE,
                uf)

        if not results_raw:
            raise NotFoundException(username)

        results = []
        for dn, attributes in results_raw:
            if dn is None:
                continue

            u = _USER(
                    dn=dn,
                    attributes=attributes,
                    groups=attributes['memberOf'])

            results.append(u)

        assert \
            len(results) == 1, \
            "More than one result was found for user [{}], which doesn't " \
            "make sense: {}".format(username, results)

        return results[0]

    def get_group_members(self, group_name):
        """Return a list of DNs."""

        gf = _GROUP_QUERY.format(group_name=group_name)
        results_raw = \
            self.__resource.search_s(
                self.__base_dn,
                ldap.SCOPE_SUBTREE,
                gf)

        if not results_raw:
            raise NotFoundException(group_name)

        collections = []
        for dn, attributes in results_raw:
            if dn is None:
                continue

            collections.append(attributes['member'])

        if not collections:
            raise NotFoundException(group_name)

        assert \
            len(collections) == 1, \
            "Too many sets of results returned: {}".format(collections)

        return collections[0]

Go: Implementing Subcommands With go-flags

github.com/jessevdk/go-flags is the go-to tool for argument processing. It supports subcommands but understanding how to do it is a feat of reverse-engineering. So, here is an example.

Code:

package main

import (
    "os"

    "github.com/jessevdk/go-flags"
)

type readParameters struct {
}

type writeParameters struct {
}

type parameters struct {
    Verbose bool `short:"v" long:"verbose" description:"Display logging"`
    Read readParameters `command:"read" alias:"r" description:"Read functions"`
    Write readParameters `command:"write" alias:"w" description:"Write functions"`
}

var (
    arguments = new(parameters)
)

func main() {
    p := flags.NewParser(arguments, flags.Default)

    _, err := p.Parse()
    if err != nil {
        os.Exit(-1)
    }

    switch p.Active.Name {
    case "read":
        //...
    case "write":
        //...
    }
}

If you were to save it as "args.go", this is what the help and the usage would look like:

$ go run args.go -h
Usage:
  args [OPTIONS] 

Application Options:
  -v, --verbose  Display logging

Help Options:
  -h, --help     Show this help message

Available commands:
  read   Read functions (aliases: r)
  write  Write functions (aliases: w)

exit status 255

$ go run args.go read