Author: dustin

Python: Parsing XML and Retaining the Comments

By default, Python’s built-in ElementTree module strips comments as it reads them. The solution is just obscure enough to be hard to find.

import xml.etree.ElementTree as ET

class _CommentedTreeBuilder(ET.TreeBuilder):
    def comment(self, data):
        self.start('!comment', {})
        self.data(data)
        self.end('!comment')

def parse(filepath):
    ctb = _CommentedTreeBuilder()
    xp = ET.XMLParser(target=ctb)
    tree = ET.parse(filepath, parser=xp)

    root = tree.getroot()
    # ...

When enumerating the parsed nodes, the comments will have a tag-name of “!comment”.

ssl: Promoting Existing Client Socket to SSL in C/C++

You may be in a situation where something else produces the sockets for you (such as an event-loop) or you otherwise need to manage the socket rather then allowing something else to.

#include <stdio.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <openssl/ssl.h>

int main(int argc, char *argv[])
{
    int sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (!sockfd) {
        printf("Error creating socket.\n");
        return -1;
    }

    struct sockaddr_in sa;
    memset (&sa, 0, sizeof(sa));

    sa.sin_family = AF_INET;
    sa.sin_addr.s_addr = inet_addr("172.217.2.196");
    sa.sin_port = htons (443); 

    socklen_t socklen = sizeof(sa);
    if (connect(sockfd, (struct sockaddr *)&sa, socklen)) {
        printf("Error connecting to server.\n");
        return -1;
    }

    SSL_library_init();
    SSLeay_add_ssl_algorithms();
    SSL_load_error_strings();

    const SSL_METHOD *meth = TLSv1_2_client_method();
    SSL_CTX *ctx = SSL_CTX_new (meth);

    SSL *ssl = SSL_new (ctx);
    if (ssl == NULL) {
        printf("Could not create SSL context.\n");
        return -1;
    }

    SSL_set_fd(ssl, sockfd);

    int err = SSL_connect(ssl);
    if (err <= 0) {
        printf("Could not connect.\n");
        return -1;
    }

    printf ("SSL connection using %s\n", SSL_get_cipher (ssl));

    // Do send/receive here.

    return 0;
}

Adapted from openssl-in-c-socket-connection-https-client, and works with both OpenSSL and BoringSSL.

Go: Parsing Time Expressions

go-time-parse will parse time expressions into time.Duration quantities. From the example:

actualDuration, phraseType, err := ParseDuration("24 days from now")
log.PanicIf(err)

fmt.Printf("%d [%s]\n", actualDuration/time.Hour/24, phraseType)

actualDuration, phraseType, err = ParseDuration("now")
log.PanicIf(err)

fmt.Printf("%d [%s]\n", actualDuration, phraseType)

actualDuration, phraseType, err = ParseDuration("12m")
log.PanicIf(err)

fmt.Printf("%d [%s]\n", actualDuration/time.Minute, phraseType)

actualDuration, phraseType, err = ParseDuration("every 6 hours")
log.PanicIf(err)

fmt.Printf("%d [%s]\n", actualDuration/time.Hour, phraseType)

Output:

24 [time]
0 [time]
12 [interval]
6 [interval]

Identifying Nearest Major Cities

go-geographic-attractor is a new project that indexes world city and population data and can match a given coordinate to either the nearest major city or the nearest city (if no major city is near) in near-instantaneous time.

From the example:

// Load countries.

countryDataFilepath := path.Join(appPath, "test", "asset", "countryInfo.txt")

f, err := os.Open(countryDataFilepath)
log.PanicIf(err)

defer f.Close()

countries, err := geoattractorparse.BuildGeonamesCountryMapping(f)
log.PanicIf(err)

// Load cities.

gp := geoattractorparse.NewGeonamesParser(countries)

cityDataFilepath := path.Join(appPath, "index", "test", "asset", "allCountries.txt.detroit_area_handpicked")
g, err := os.Open(cityDataFilepath)
log.PanicIf(err)

defer g.Close()

ci := NewCityIndex()

err = ci.Load(gp, g)
log.PanicIf(err)

// Do the query.

clawsonCoordinates := []float64{42.53667, -83.15041}

sourceName, visits, cr, err := ci.Nearest(clawsonCoordinates[0], clawsonCoordinates[1])
log.PanicIf(err)

// Print the results.

for _, vhi := range visits {
    fmt.Printf("%s: %s\n", vhi.Token, vhi.City)
}

fmt.Printf("\n")

fmt.Printf("Source: %s\n", sourceName)
fmt.Printf("ID: %s\n", cr.Id)
fmt.Printf("Country: %s\n", cr.Country)
fmt.Printf("City: %s\n", cr.City)
fmt.Printf("Population: %d\n", cr.Population)
fmt.Printf("Latitude: %.10f\n", cr.Latitude)
fmt.Printf("Longitude: %.10f\n", cr.Longitude)

Go: Generating KML

Use go-kml to build a KML structure. Then, marshal using the xml package.

An example using code from the project:

package main

import (
    "encoding/xml"
    "os"

    "github.com/twpayne/go-kml"
)

func main() {
    x := kml.KML(
        kml.Placemark(
            kml.Name("The Pentagon"),
            kml.Polygon(
                kml.Extrude(true),
                kml.AltitudeMode("relativeToGround"),
                kml.OuterBoundaryIs(
                    kml.LinearRing(
                        kml.Coordinates([]kml.Coordinate{
                            {-77.05788457660967, 38.87253259892824, 100},
                            {-77.05465973756702, 38.87291016281703, 100},
                            {-77.05315536854791, 38.87053267794386, 100},
                            {-77.05552622493516, 38.868757801256, 100},
                            {-77.05844056290393, 38.86996206506943, 100},
                            {-77.05788457660967, 38.87253259892824, 100},
                        }...),
                    ),
                ),
            ),
        ),
    )

    e := xml.NewEncoder(os.Stdout)
    e.Indent("", "  ")

    err := e.Encode(x)
    if err != nil {
        panic(err)
    }
}

This prints:

<kml xmlns="http://www.opengis.net/kml/2.2">
  <Placemark>
    <name>The Pentagon</name>
    <Polygon>
      <extrude>1</extrude>
      <altitudeMode>relativeToGround</altitudeMode>
      <outerBoundaryIs>
        <LinearRing>
          <coordinates>-77.05788457660967,38.87253259892824,100 -77.05465973756702,38.87291016281703,100 -77.0531553685479,38.87053267794386,100 -77.05552622493516,38.868757801256,100 -77.05844056290393,38.86996206506943,100 -77.05788457660967,38.87253259892824,100</coordinates>
        </LinearRing>
      </outerBoundaryIs>
    </Polygon>
  </Placemark>
</kml>

..which renders as this when using Google Earth:

Google Earth Pro_004

Use ADB to Connect to Your Android Device From a Docker Container

You may have a use-case where you want to write software to manipulate an Android device using a system or set of tools that are not natively available from your current system. However, you might be able to expose this as a Docker image. For example, your device is (or will be) connected to a Windows machine and you really want to or need to use Linux tools.

No problem. ADB implicitly uses a client-server model: The ADB tool (on your system) connects to the ADB server (runs in the background on your system) which interacts with the ADB daemon (runs on your device). This means that we can forward requests from ADB on the command-line in the guest container in Docker to the ADB server on the host system.

The ADB client and server have to be at the same version, or the client will indiscriminately kill/restart your ADB server. So, as I am currently running Ubuntu 14.04 on my host system, I will do the same in Docker.

First, I will make sure the ADB server is running on my host system. Most of the subcommands that will automatically start the local server, but I will start it directly:

$ adb start-server
* daemon not running. starting it now on port 5037 *
* daemon started successfully *

Now, I will start a container in Docker with Ubuntu 14.04 and automatically install ADB before dropping to a prompt. Note that we are passing “–network=host” in order to share the host’s network identity:

$ docker run -i -t --network=host ubuntu:14.04 /bin/bash -c "sudo apt-get update && sudo apt-get install -y android-tools-adb && /bin/bash"

Eventually, you will end-up at the prompt. Just do something simple like enumerating the devices:

root@mlll2664:/# adb devices
List of devices attached 
05157df572841820 device

The “mlll2664” hostname, represented in the prompt in the Docker container, is, actually, the same hostname as my host system.

So, there you go. Not too painful.