clean-up

The netem file won't work, but I'll clean that up when it's needed.

README

@@ -16,3 +16,14 @@ Before running start_testbed.py, you must run:
 
 This only needs to be done once for your VM.
 
+To start the testbed, run
+
+    ./start_testbed.py [-f <config file>]
+
+To stop the testbed, run
+
+    ./stop_testbed.py [-f <config file>]
+
+In both cases, if the config file isn't specified, it will use
+example.yml
+

host_vars/localhost

----
-runtime_repo: https://github-enterprise.px.ftw/execution/runtime.git
-runtime_version: master
-runtime_dir: /opt/runtime
-controller_repo: https://github-enterprise.px.ftw/Controller/BlueController.git
-controller_version: master
-controller_dir: /usr/local/src/BlueController
-playbook_repo: https://github-enterprise.px.ftw/plan-construction/playbook.git
-playbook_version: master
-plan_dir: /opt/plan
-pip_cachedir: /opt/pipcache
-pip_args: "--no-index -f {{pip_cachedir}}/"
-nsq_uri: http://ns1.px.ftw/~bdennis/nsq-0.2.31.linux-amd64.go1.3.tar.gz
-nsq_unpacked: nsq-0.2.31.linux-amd64.go1.3
-nsq_dir: /opt/nsq
-docker_registry: docker.px.ftw
-sshd_image: "{{docker_registry}}/base/sshd:latest"
-luna_image: "{{docker_registry}}/px_core/px-luna:latest"
-blue_builder_image: "{{docker_registry}}/px_controller/blue_builder:latest"
-grey_builder_image: "{{docker_registry}}/px_controller/grey_builder:latest"
-subnet: 186.192.0.0
-subnet_prefix: 16
-gw_ip: 186.192.0.0
-plan_master_ip: 186.192.0.1
-luna_ip: 186.192.0.2
-nsq_ip: 127.0.0.1
-gwname: gw
-bridgedev: baconet0
-# If we're bridging outside of this docker cloud, the larger subnet's prefix
-wsubnet_prefix: 0
-wsubnet: 186.192.0.0
-i_wlink: pancetta1
-e_wlink: pancetta0
-have_WAS: "{{wsubnet_prefix>0}}"
-detached: false
-git_update: "{{'no' if detached|bool else 'yes'}}"
-keep_builder: false
-# Pre-deployment doesn't fully work at the moment.  We'll revisit the default when it does.
-deployer: true
-# This should be changed to "false" once the latest version on pypi-internal is reliably good.
-reinstall_runtime: true
-capture_bridges: false

library/extract_routing

-#! /usr/bin/env python
-
-import json
-
-def uuid_add(uuid,idx):
-    uuid_pieces = uuid.split('-')
-    last_one = 0xffffffffffff & (int(uuid_pieces[-1],base=16) + idx)
-    uuid_pieces[-1] = "%0x" % last_one
-    return '-'.join(uuid_pieces)
-
-def collapse_subnets(full_list):
-    parsed_out = map( lambda x: (x,cidr_to_hex(x)), full_list )
-    rearranged = map( lambda x: (x[1][0] , (x[1][1],x[0])), parsed_out )
-    full_map = dict(rearranged)
-    base_addrs = full_map.keys()
-    base_addrs.sort()
-    removals = []
-    for i in range(len(base_addrs)):
-        cur_addr = base_addrs[i]
-        cur_data = full_map[cur_addr]
-        cur_max = cur_data[0]
-        for j in range(i):
-            prev_addr = base_addrs[j]
-            prev_data = full_map[prev_addr]
-            prev_max = prev_data[0]
-            if cur_addr < prev_addr: continue
-            if cur_addr > prev_max: continue
-            if cur_max > prev_max: continue
-            removals.append(cur_addr)
-            break
-    for addr in removals:
-        full_map.pop(addr)
-    subnets = map(lambda x: full_map[x][1], full_map)
-    return subnets
-
-def cidr_to_hex(cidr):
-    base_addr, mask_len = cidr.split("/")
-
-    mask = 0xffffffff & ( 0xffffffff << (32-int(mask_len)) )
-
-    q1,q2,q3,q4 = base_addr.split(".")
-    base_naddr = (int(q1)<<24) + (int(q2)<<16) + (int(q3)<<8) + int(q4)
-    # Make sure the base address is the base of the block (ie, valid CIDR)
-    base_naddr = base_naddr & mask
-    max_addr = base_naddr + (2 ** (32-int(mask_len))) - 1
-    return base_naddr,max_addr
-
-def hex_to_quads(hex_addr):
-    q1 = (hex_addr & 0xff000000) >> 24
-    q2 = (hex_addr & 0xff0000) >> 16
-    q3 = (hex_addr & 0xff00) >> 8
-    q4 = (hex_addr & 0xff)
-    return "%d.%d.%d.%d" % (q1,q2,q3,q4)
-
-def extract(ntwk,cidr,root,root_bridge,root_address,uplist=[]):
-    if 'name' in ntwk:
-        name = ntwk['name']
-    else:
-        module.fail_json(msg="Missing 'name' attribute")
-    networks = []
-    gw_name = name + "0"
-    current = { "name": name }
-    if 'subnet' not in ntwk:
-        module.fail_json(msg="Missing 'subnet' attribute")
-    current["top"] = cidr
-    current["subnet"] = ntwk['subnet']
-    current["upward"] = collapse_subnets( [ cidr, current["subnet"] ] + uplist )
-    addr_base, max_addr = cidr_to_hex(current["subnet"])
-    gw_addr = addr_base + 0
-    bridge_dev = "ctr_b%0X" % gw_addr
-
-    if 'nodes' in ntwk:
-        nodes = ntwk['nodes']
-
-        if 'num' not in nodes:
-            module.fail_json(msg="Missing 'num' attribute")
-        num_nodes = nodes['num']
-        if addr_base + num_nodes > max_addr:
-            module.fail_json(
-                msg="Requested %d nodes, but address space is too small" % num_nodes
-            )
-
-        current["nodes"] = []
-        for idx in range(num_nodes):
-            node_idx = idx+1
-            addr = addr_base + node_idx
-            node_data = {
-                "name" : name + str(node_idx),
-                "address" : hex_to_quads(addr),
-                "internal_device" : "ctr_i%0X" % addr,
-                "external_device" : "ctr_e%0X" % addr,
-                "route_up" : current["upward"],
-                "gateway" : gw_name,
-                "bridge" : bridge_dev,
-                "gateway_address" : hex_to_quads(gw_addr),
-                "subnet" : current["subnet"]
-            }
-            if 'uuid' in nodes:
-                node_data["uuid"] = uuid_add(nodes["uuid"],node_idx)
-            current["nodes"].append(node_data)
-            for k in nodes:
-                if k == 'num': continue
-                if k == 'uuid': continue
-                node_data[k] = nodes[k]
-
-    if 'networks' in ntwk:
-        current["downstreams"] = []
-        for subnet in ntwk['networks']:
-            g,d = extract(subnet,cidr,
-                          gw_name,bridge_dev,hex_to_quads(gw_addr),
-                          uplist=current["upward"])
-            networks.extend(g)
-            downstream = {}
-            downstream["name"] = d["name"]
-            if 'downstreams' in d:
-                ds_reach = map(lambda x: x["reachable"], d["downstreams"])
-                ds_ntwks = [ d["subnet"] ]
-                for x in ds_reach:
-                    ds_ntwks.extend(x)
-                downstream["reachable"] = collapse_subnets( ds_ntwks )
-            else:
-                downstream["reachable"] = [ d["subnet"] ]
-            current["downstreams"].append(downstream)
-    if 'downstreams' in current:
-        ds_reach = map(lambda x: x["reachable"], current["downstreams"])
-        ds_ntwks = [ current["subnet"] ]
-        for x in ds_reach:
-            ds_ntwks.extend( x )
-        current["reachable"] = collapse_subnets( ds_ntwks )
-    else:
-        current["reachable"] = [ current["subnet"] ]
-    current["upstream"] = root
-    current["upstream_bridge"] = root_bridge
-    current["upstream_address"] = root_address
-    current["gateway"] = {
-        "name" : gw_name,
-        "address" : hex_to_quads(gw_addr),
-        "bridge" : bridge_dev,
-        "internal_device" : "ctr_i%0X" % gw_addr,
-        "external_device" : "ctr_e%0X" % gw_addr,
-        "upstream" : current["upstream"],
-        "upstream_bridge" : current["upstream_bridge"],
-        "upstream_address" : current["upstream_address"],
-        "route_down" : current["reachable"],
-        "route_up" : current["upward"],
-    }
-    networks.append(current)
-    return (networks,current)
-
-def main():
-    module = AnsibleModule(argument_spec=dict(
-        network=dict(required=True),
-        cidr=dict(required=True),
-        root=dict(required=True),
-        root_bridge=dict(required=True),
-        root_address=dict(required=True),
-    ))
-
-    changed=False
-    network = module.params['network']
-    cidr = module.params['cidr']
-    root = module.params['root']
-    root_bridge = module.params['root_bridge']
-    root_address = module.params['root_address']
-    js = network.replace('\'','"')
-    j = json.loads(js)
-    netlist, top = extract(j,cidr,root,root_bridge,root_address)
-    netlist_tuples = map(lambda x: (x["name"],x), netlist)
-    netmap = dict(netlist_tuples)
-    gateways = []
-    nodes = []
-    for ntwk_name in netmap:
-        ntwk = netmap[ntwk_name]
-        gateways.append(ntwk["gateway"])
-        if 'nodes' in ntwk:
-            nodes.extend(ntwk["nodes"])
-    top_dict = {
-        "name" : top["name"],
-        "gateway" : top["gateway"]["name"],
-        "bridge" : top["gateway"]["bridge"],
-        "address" : top["gateway"]["address"],
-        "external_device" : top["gateway"]["external_device"],
-        "internal_device" : top["gateway"]["internal_device"]
-    }
-    reachable = top["reachable"]
-
-    module.exit_json(changed=changed,networks=netmap,top=top_dict,gateways=gateways,nodes=nodes,reachable=reachable)
-
-from ansible.module_utils.basic import *
-main()

library/networked

-#! /bin/bash
-
-arg_file=$1
-
-hexify() {
-    echo $1 | awk -F . '{printf "%02X%02X%02X%02X", $1, $2, $3, $4}'
-}
-
-make_quad() {
-    numeric=$1
-    q1=$(( ($numeric>>24) & 0xff ))
-    q2=$(( ($numeric>>16) & 0xff ))
-    q3=$(( ($numeric>>8) & 0xff ))
-    q4=$(( ($numeric) & 0xff ))
-    echo "$q1.$q2.$q3.$q4"
-}
-
-make_context() {
-    if [ $# > 0 ]
-    then
-	if [ "x$1" != "x" ]
-	then
-            echo "ip netns exec $1"
-	else
-	    echo ""
-	fi
-    else
-        echo ""
-    fi
-}
-
-have_device() {
-    local dev=$1
-    if [ $# > 1 ]
-    then
-	local ns=$2
-    fi
-    if [ "x${ns}" == "x" ]
-    then
-	unset -v ns
-    fi
-    local ctx=$(make_context $ns)
-    echo "looking for ${dev} (${ctx})" >>${log}
-    ${ctx} ip link show ${dev} >>${log} 2>&1
-    return $?
-}
-
-# This has some Byzantine fact handling, since we store things in a
-# global variable.  That means we have to move it aside when we'd be
-# adding things that need to appear multiple times, so that we can
-# encapsulate them.
-do_full() {
-    local name=$1;shift
-    local ip=$1;shift
-    local gw=$1;shift
-    local gwdev=$1;shift
-    local subnet=$1;shift
-    local prefix=$1;shift
-    local i_dev=$1;shift
-    local e_dev=$1;shift
-    local gwns=$1;shift
-
-    local c_name="\"name\":\"${name}\""
-    local c_type="\"type\":\"${type}\""
-    local c_addr="\"addr\":\"${ip}\""
-    local c_idev="\"internal_device\":\"${i_dev}\""
-    local c_edev="\"external_device\":\"${e_dev}\""
-    local ctr_info="${c_name},${c_type},${c_addr},${c_idev},${c_edev}"
-
-    local oldfacts=${facts}
-    facts=""
-    do_link ${i_dev} ${ip} ${name} ${e_dev} ${gw} ${gwns}
-    local retval=$?
-    local linkfacts=${facts}
-    facts=${oldfacts}
-    local ctr_info="${ctr_info},\"link\":{${linkfacts}}"
-    if [ 0 != $retval ]
-    then
-        add_fact "container" "{${ctr_info}}"
-        return 1
-    fi
-
-    local i_ctx=$(make_context ${name})
-    local e_ctx=$(make_context ${gwns})
-
-    local oldfacts=${facts}
-    facts=""
-    do_route ${i_dev} ${ip} "${subnet}/${prefix}" ${name}
-    local retval=$?
-    local routefacts=${facts}
-    facts=${oldfacts}
-    local rfactsarray="[{${routefacts}}"
-    if [ 0 != $retval ]
-    then
-        local ctr_info="${ctr_info},\"routes\":${rfactsarray}]"
-        add_fact "container" "{${ctr_info}}"
-        return 1
-    fi
-    local oldfacts=${facts}
-    facts=""
-    do_route ${e_dev} ${gw} ${ip}/32 ${gwns}
-    local retval=$?
-    local routefacts=${facts}
-    facts=${oldfacts}
-    local rfactsarray="${rfactsarray},{${routefacts}}]"
-    local ctr_info="${ctr_info},\"routes\":${rfactsarray}"
-    if [ 0 != $retval ]
-    then
-        add_fact "container" "{${ctr_info}}"
-        return 1
-    fi
-
-    local oldfacts=${facts}
-    facts=""
-    do_netem ${e_dev} ${gwns}
-    local retval=$?
-    local netemfacts=${facts}
-    facts=${oldfacts}
-    local nfactsarray="[{${netemfacts}}"
-    if [ 0 != $retval ]
-    then
-        local ctr_info="${ctr_info},\"qdiscs\":${nfactsarray}]"
-        add_fact "container" "{${ctr_info}}"
-        return 1
-    fi
-    local oldfacts=${facts}
-    facts=""
-    do_netem ${i_dev} ${name}
-    local retval=$?
-    local netemfacts=${facts}
-    facts=${oldfacts}
-    local nfactsarray="${nfactsarray},{${netemfacts}}]"
-    local ctr_info="${ctr_info},\"qdiscs\":${nfactsarray}"
-    if [ 0 != $retval ]
-    then
-        add_fact "container" "{${ctr_info}}"
-        return 1
-    fi
-    if [ "x${bandwidth}" != "x" ]
-    then
-        ${i_ctx} tc qdisc change dev ${i_dev} root netem rate ${bandwidth} >>${log} 2>&1
-        add_fact "bandwidth" "\"${bandwidth}\""
-    fi
-
-    local oldfacts=${facts}
-    facts=""
-    do_addif ${e_dev} ${gwdev} ${gwns}
-    local retval=$?
-    local addiffacts=${facts}
-    facts=${oldfacts}
-    local ctr_info="${ctr_info},\"bridging\":{${addiffacts}}"
-    if [ 0 != $retval ]
-    then
-        add_fact "container" "{${ctr_info}}"
-        return 1
-    fi
-
-    add_fact "container" "{${ctr_info}}"
-    return 0
-}
-
-do_was() {
-    local hexgw=$1
-    local wprefix=$2
-    local name=$3
-    local i_dev=$4
-    local ip=$5
-
-    wlength=$(( 32 - ${wprefix} ))
-    wsubnet=$(make_quad $(( 0x${hexgw} >> $wlength << $wlength )))
-    local oldfacts=${facts}
-    facts=""
-    do_route ${i_dev} ${ip} "${wsubnet}/${wprefix}" ${name}
-    local retval=$?
-    local routefacts=${facts}
-    facts=${oldfacts}
-    add_fact "external_subnet" "{${routefacts}}"
-
-    return ${retval}
-}
-
-do_link() {
-    local dev0=$1
-    local addr0=$2
-    local ns0=$3
-    local dev1=$4
-    local addr1=$5
-    local ns1=$6
-
-    if [ "x${ns0}" == "x" ]
-    then
-        unset -v ns0
-    fi
-    if [ "x${ns1}" == "x" ]
-    then
-        unset -v ns1
-    fi
-
-    local ctx0=$(make_context ${ns0})
-    local ctx1=$(make_context ${ns1})
-
-    have_device $dev0 $ns0
-    local have_dev0=$?
-    have_device $dev1 $ns1
-    local have_dev1=$?
-
-    make_ep_json() {
-        local json="{"
-        local json="${json}\"addr\":\"$1\","
-        local json="${json}\"dev\":\"$2\""
-        if [ $# > 2 ]
-        then
-            local json="${json},\"ns\":\"$3\""
-        fi
-        local json="${json}}"
-        echo $json
-    }
-    local ep0=$(make_ep_json $addr0 $dev0 $ns0)
-    local ep1=$(make_ep_json $addr1 $dev1 $ns1)
-    add_fact "endpoints" "[${ep0},${ep1}]"
-
-    # Idempotent link add
-    if [ \( 0 -eq ${have_dev0} \) -a \( 0 -eq ${have_dev1} \) ]
-    then
-        /bin/true
-    elif [ 0 -eq ${have_dev0} ]
-    then
-        err="device ${dev0} already exists"
-        add_fact "linked" "false"
-        return 1
-    elif [ 0 -eq ${have_dev1} ]
-    then
-        err="device ${dev1} already exists"
-        add_fact "linked" "false"
-        return 1
-    else
-        is_changed="true"
-        ip link add ${dev0} type veth peer name ${dev1} >>${log} 2>&1
-        if [ $? -ne 0 ]
-        then
-            err="cannot create devices ${dev0} and ${dev1}"
-            return 1
-        fi
-        if [ "x${ns0}" != "x" ]
-        then
-            ip link set ${dev0} netns ${ns0} >>${log} 2>&1
-            if [ $? -ne 0 ]
-            then
-		err="cannot put ${dev0} in namespace ${ns0}"
-		return 1
-            fi
-        fi
-        if [ "x${ns1}" != "x" ]
-        then
-            ip link set ${dev1} netns ${ns1} >>${log} 2>&1
-            if [ $? -ne 0 ]
-            then
-		err="cannot put ${dev1} in namespace ${ns1}"
-		return 1
-            fi
-        fi
-    fi
-
-    # Setting the link address is idempotent already.
-    $ctx0 ip addr add $addr0/32 dev $dev0 >>${log} 2>&1
-    $ctx1 ip addr add $addr1/32 dev $dev1 >>${log} 2>&1
-
-    # Bringing up a link is idempotent already.
-    $ctx0 ip link set $dev0 up >>${log} 2>&1
-    $ctx1 ip link set $dev1 up >>${log} 2>&1
-
-    add_fact "linked" "true"
-    return 0
-}
-
-do_bridge() {
-    local dev=$1; shift
-    local addr=$1; shift
-    if [ $# > 0 ]
-    then
-        local ns=$1; shift
-    fi
-    local ctx=$(make_context $ns)
-
-    local have_bridge="no"
-    local bridges=$($ctx brctl show | tail -n +1 | cut -f 1)
-    for b in ${bridges}
-    do
-        if [ $b == $dev ]
-        then
-            local have_bridge="yes"
-        fi
-    done
-
-    if [ ${have_bridge} == "no" ]
-    then
-        $ctx brctl addbr $dev >>${log} 2>&1
-	$ctx ip address add dev $dev local ${addr} >>${log} 2>&1
-        is_changed="true"
-    fi
-
-    # This is already idempotent
-    $ctx ip link set dev $dev up >>${log} 2>&1
-
-    local br_info="\"dev\":\"${dev}\""
-    if [ "x${ns}" != "x" ]
-    then
-        local br_info="${br_info},\"ns\":\"${ns}\""
-    fi
-    add_fact "bridge" "{${br_info}}"
-    return 0
-}
-
-do_addif() {
-    local dev=$1
-    local bridge=$2
-    if [ $# > 2 ]
-    then
-        local ns=$3
-    fi
-    local ctx=$(make_context $ns)
-
-    # Check to see if the device is already using that bridge.
-    $ctx ip link show $dev | grep -E "\s${bridge}\s"
-    if [ 0 -ne $? ]
-    then
-        $ctx brctl addif $bridge $dev >>${log} 2>&1
-        is_changed="true"
-    fi
-
-    local dev_info="\"dev\":\"${dev}\""
-    local bridge_info="\"bridge\":\"${bridge}\""
-    if [ "x${ns}" != "x" ]
-    then
-        local ns_info="\"ns\":\"${ns}\""
-    fi
-    local if_info="${dev_info},${bridge_info}${ns_info:+,}${ns_info}"
-    add_fact "addif" "{${if_info}}"
-    return 0
-}
-
-do_route() {
-    local dev=$1
-    local addr=$2
-    local net=$3
-    if [ $# > 3 ]
-    then
-        local ns=$4
-    fi
-    local info_dev="\"dev\":\"${dev}\""
-    local info_addr="\"from\":\"${addr}\""
-    local info_net="\"to\":\"${net}\""
-    if [ "x${ns}" != "x" ]
-    then
-        local info_ns=",\"ns\":\"${ns}\""
-    fi
-    local ctx=$(make_context $ns)
-    $ctx ip route list match | grep $dev
-    if [ 0 -ne $? ]
-    then
-        local msg=$($ctx ip route add $net dev $dev proto static scope global src $addr 2>&1)
-        if [ 0 -ne $? ]
-        then
-            err="failed to add route: ${msg}"
-            add_fact "route" "{${info_dev},${info_addr},${info_net}${info_ns},\"success\":false}"
-            return 1
-        fi
-        is_changed="true"
-    fi
-
-    add_fact "route" "{${info_dev},${info_addr},${info_net}${info_ns},\"success\":true}"
-    return 0
-}
-
-do_netem() {
-    local dev=$1
-    if [ $# > 1 ]
-    then
-        local ns=$2
-    fi
-    local ctx=$(make_context $ns)
-    $ctx ip link show $dev | grep -E "\snetem\s"
-    if [ 0 -ne $? ]
-    then
-        $ctx tc qdisc add dev $dev root handle 1:0 netem >>${log} 2>&1
-        is_changed="true"
-    fi
-
-    local info_dev="\"dev\":\"${dev}\""
-    if [ "x${ns}" != "x" ]
-    then
-        local info_ns=",\"ns\":\"${ns}\""
-    fi
-    add_fact "netem" "{${info_dev}${info_ns}}"
-
-    return 0
-}
-
-add_fact() {
-    local fact_name=$1
-    local fact_value=$2
-    facts="${facts}${facts:+,}\"${fact_name}\":${fact_value}"
-}
-
-parse_args() {
-    local state="full"
-    arg_file=$1
-    for line in $(cat ${arg_file})
-    do
-        local $line
-    done
-    case "${state}" in
-        "full")
-            local hexaddr=$(hexify ${ip})
-            local i_dev="ctr_i${hexaddr}" # Internal -- container-side
-            local e_dev="ctr_e${hexaddr}" # External -- gateway-side
-
-            local hexgw=$(hexify ${gw})
-            local length=$(( 32 - ${prefix} ))
-            local subnet=$(make_quad $(( 0x${hexgw} >> $length << $length )))
-
-            do_full ${name} ${ip} ${gw} ${gwdev} \
-                ${subnet} ${prefix} ${i_dev} ${e_dev} ${gwns}
-            if [ 0 -ne $? ]
-            then
-                return 1
-            fi
-            if [ ${wprefix} -gt 0 ]
-            then
-                do_was ${hexgw} ${wprefix} ${name} ${i_dev} ${ip}
-            else
-                return 0
-            fi
-            ;;
-        "link")
-            do_link $dev0 $addr0 ${ns0:-""} $dev1 $addr1 ${ns1:-""}
-            return $?
-            ;;
-        "bridge")
-            do_bridge $dev $addr $ns
-            return $?
-            ;;
-        "addif")
-            do_addif $dev $bridge $ns
-            return $?
-            ;;
-        "route")
-            do_route $dev $addr $net $ns
-            return $?
-            ;;
-        "netem")
-            do_netem $dev $ns
-            return $?
-            ;;
-    esac
-}
-
-create_json() {
-    local failed=$1
-    if [ $# > 1 ]
-    then
-        local msg=$2
-    fi
-
-    local json="{"
-    local json="${json} \"failed\" : ${failed}"
-    local json="${json}, \"changed\" : ${is_changed}"
-    local json="${json}, \"data\" : { ${facts} }"
-    if [ "x${msg}" != "x" ]
-    then
-        local json="${json}, \"msg\" : \"${msg}\""
-    fi
-    local json="${json} }"
-    echo ${json}
-}
-
-facts=""
-err=""
-is_changed="false"
-log=$(mktemp)
-parse_args ${arg_file}
-if [ 0 -eq $? ]
-then
-    echo $(create_json "false")
-    exit 0
-else
-    echo $(create_json "true" "${err}")
-    exit 1
-fi

library/subnet_validate

-#! /usr/bin/env python
-
-def main():
-    module = AnsibleModule(argument_spec=dict(
-            subnet=dict(required=True,type='str'),
-            num=dict(required=True,type='int'),
-            start_index=dict(type='int',default=1),
-    ))
-
-    addr, prefix = module.params['subnet'].split('/')
-    aquads = addr.split('.')
-    naddr = (
-        int(aquads[0]) * (2 ** 24) +
-        int(aquads[1]) * (2 ** 16) +
-        int(aquads[2]) * (2 ** 8) +
-        int(aquads[3])
-    )
-    bits_avail = 32-int(prefix)
-    mask = 0xffffffff << bits_avail
-    nbase = naddr & mask
-    quads = (
-        nbase / (2 ** 24),
-        (nbase / (2 ** 16)) % 256,
-        (nbase / (2 ** 8)) % 256,
-        nbase % 256
-    )
-    max_num = 2 ** bits_avail - module.params['start_index']
-
-    if module.params['num'] > max_num:
-        module.fail_json(msg="Number of hosts exceeds subnet size")
-
-    module.exit_json(changed=False, quads=quads, naddr=naddr, mask=mask, nbase=nbase)
-
-from ansible.module_utils.basic import *
-main()

library/tcpdump

-#! /usr/bin/env python
-
-import os
-
-def daemon_spawn(module,namespace,file,device):
-    import os
-    import sys
-    
-    try:
-        pid = os.fork()
-    except OSError, e:
-        module.fail_json(msg="Exception {}: {}".format(e.errno,e.strerror))
-    if 0 == pid:
-        os.setsid()
-        try:
-            pid = os.fork()
-        except OSError, e:
-            os._exit(1)
-        if 0 != pid:
-            os._exit(0)
-    else:
-        module.exit_json(changed=True)
-    
-    maxfd = 2
-    for fd in range(0, maxfd):
-        try:
-            os.close(fd)
-        except OSError:
-            pass
-    os.open(os.devnull, os.O_RDWR)
-    os.open("{}.log".format(file),os.O_RDWR|os.O_CREAT)
-    os.dup2(1, 2)
-    
-    os.system("ip netns exec {} tcpdump -w {} -i {}".format(namespace,file,device))
-    os._exit(0)
-
-def main():
-    module = AnsibleModule(argument_spec=dict(
-        namespace=dict(required=True),
-        file=dict(required=True),
-        device=dict(required=True),
-    ))
-    
-    changed=True
-    namespace = module.params['namespace']
-    file = module.params['file']
-    device = module.params['device']
-    daemon_spawn(module,namespace, file, device)
-
-from ansible.module_utils.basic import *
-main()

load_topology.yml

----
-- name: Link the Luna config from the plan directory to the controller tools directory
-  file: src={{plan_dir}}/pxluna_configs_mte.py
-        dest={{controller_dir}}/tools/pxluna_configs_mte.py
-        state=link
-
-- name: Inject the node data into the core
-  shell: "./LoadPredeployedNodesIntoCore.py --nodeFile={{plan_dir}}/nodes.json --luna=pxluna_configs_mte.py --exectraceid=bogus > /var/log/bluecontroller/nodedata.stdout 2> /var/log/bluecontroller/nodedata.stderr chdir={{controller_dir}}/tools"

roles/gateway/tasks/gateway.yml

----
-# We apparently need to do this in order to convert the gateway
-# variable into something other than a string. No, it doesn't make
-# sense to me, either.
-- set_fact:
-    this: "{{gateway}}"
-
-- name: Create the gateway container
-  docker: image={{sshd_image}}
-          name={{this.name}}
-          hostname={{this.name}}
-          password=px
-          publish_all_ports=true
-          volumes="{{work_dir}}/logs/{{this.name}}:/var/log/supervisor"
-  register: gateway_ctr
-  until: not gateway_ctr.failed
-  retries: 3
-
-- set_fact:
-    ctr: "{{gateway_ctr.ansible_facts.docker_containers[0]}}"
-
-- name: Create the gateway namespaces
-  file: state=link
-        src=/proc/{{ctr.State.Pid}}/ns/net
-        path=/var/run/netns/{{this.name}}
-
-- name: Create the gateway bridges
-  networked: state=bridge
-             ns={{this.name}}
-             dev={{this.bridge}}
-             addr={{this.address}}
-
-- name: Create the gateway link to its upstream
-  networked: state=link
-             ns0={{this.name}}
-             dev0={{this.internal_device}}
-             addr0={{this.address}}
-             ns1={{this.upstream}}
-             dev1={{this.external_device}}
-             addr1={{this.upstream_address}}
-
-- name: Add network emulation to the gateway internal device
-  networked: state=netem
-             dev={{this.internal_device}}
-             ns={{this.name}}
-
-- name: Add network emulation to the gateway external device
-  networked: state=netem
-             dev={{this.external_device}}
-             ns={{this.upstream}}
-
-- name: Set up the incoming routing for the gateway
-  networked: state=route
-             ns={{this.upstream}}
-             dev={{this.external_device}}
-             addr={{this.upstream_address}}
-             net={{item}}
-  with_items: this.route_down
-
-- name: Add the incoming gateway link to the bridge
-  networked: state=addif
-             ns={{this.upstream}}
-             dev={{this.external_device}}
-             bridge={{this.upstream_bridge}}
-
-- name: Set up the outgoing routing on the gateway
-  networked: state=route
-             ns={{this.name}}
-             dev={{this.internal_device}}
-             addr={{this.address}}
-             net={{item}}
-  with_items: this.route_up
-
-- name: Add the outgoing gateway link to the bridge
-  networked: state=addif
-             ns={{this.name}}
-             dev={{this.internal_device}}
-             bridge={{this.bridge}}
-
-- name: Capture the bridge device traffic
-  tcpdump: namespace={{this.name}}
-           file="pcaps/{{this.name}}.pcap"
-           device={{this.bridge}}
-  when: capture_bridges

roles/gateway/tasks/main.yml

----
-- include: gateway.yml

roles/load_topology/tasks/main.yml

----
-- include: load_topology.yml

roles/networked/tasks/main.yml

----
-- include: networked.yml

roles/networked/tasks/networked.yml

----
-# We apparently need to do this in order to convert the node
-# variable into something other than a string. No, it doesn't make
-# sense to me, either.
-- set_fact:
-    this: "{{node}}"
-
-- name: Spin up the node
-  docker: image={{image}}
-          name={{this.name}}
-          hostname={{this.name}}
-          password=px
-          publish_all_ports=true
-          command="{{docker_command}}"
-          memory_limit=512MB
-          volumes="{{work_dir}}/logs/{{this.name}}:/var/log/supervisor,{{work_dir}}/extsrc/pipcache:{{pip_cachedir}}"
-  register: node_docker_info
-  until: not node_docker_info.failed
-  retries: 3
-
-- set_fact:
-    node_ctr: "{{node_docker_info.ansible_facts.docker_containers[0]}}"
-
-- name: Create the network namespace
-  file: state=link
-        src=/proc/{{node_ctr.State.Pid}}/ns/net
-        path=/var/run/netns/{{this.name}}
-
-- name: Create the link to the gateway
-  networked: state=link
-             ns0={{this.name}}
-             dev0={{this.internal_device}}
-             addr0={{this.address}}
-             ns1={{this.gateway}}
-             dev1={{this.external_device}}
-             addr1={{this.gateway_address}}
-
-- name: Create routes from the node to the gateway
-  networked: state=route
-             ns={{this.name}}
-             dev={{this.internal_device}}
-             addr={{this.address}}
-             net={{item}}
-  with_items: this.route_up
-
-- name: Create the route from the gateway to the node
-  networked: state=route
-             ns={{this.gateway}}
-             dev={{this.external_device}}
-             addr={{this.gateway_address}}
-             net={{this.address}}/32
-
-- name: Add network emulation to the internal device
-  networked: state=netem
-             dev={{this.internal_device}}
-             ns={{this.name}}
-
-- name: Add network emulation to the external device
-  networked: state=netem
-             dev={{this.external_device}}
-             ns={{this.gateway}}
-
-- name: Add the external interface to the bridge
-  networked: state=addif
-             ns={{this.gateway}}
-             dev={{this.external_device}}
-             bridge={{this.bridge}}
-
-- name: Configure the network emulation for the node
-  netem: state='{{this.link|to_json}}'
-         name={{this.name}}
-         device={{this.internal_device}}
-  when: this.link is defined

roles/store_output/defaults/main.yml

-other_nodes_data: {}
-planx_key: |
-  "-----BEGIN RSA PRIVATE KEY-----
-  MIIEowIBAAKCAQEAnc1GKcTn7AdgFH8M6TJf+aZGg6Iem8x3E3QXAsUN6i/hYqxL
-  tmWMg3Btlsw5ibsNV6ACJmPpZop7YrAr/3Ns/XFmUQKGGNwDGCVALSpbtDElq37k
-  j5JwVT2QrBrCO2auaXENZ+QOua8lkR7i6IDDBG/3HKOLIwWIEv5JWn76tMwU3gy2
-  On8udYWLFAcVwCOX/sMRWOjWS18f3gVivgtXlGMr+ZELbu6VUy/k356aWVoMKOBC
-  Ij94W2vHEI88NJJDl7NVV3hBEtisAk9PeMAbdqRLWcDtnUT/OWv/Lzwg1tzEnO7z
-  fL+q+kuTNGkYx/+CyPM37jKKgCLvUp+bnsOfuQIDAQABAoIBAGLKcWOS73MXxvHt
-  /GI99FHx4/Fi3HDDV8TpB8LnJOf5I8xHe3rsJkTfbOxcXKU6Yob6GH56TFOy3L4S
-  9uiNFa2xgdGzl8oWayWBpKTiANlVb3TnEyZDEmp7CYiFlBXmGcjPSwpAf2uGGCm7
-  QWk3u1+azQSz6lZ5jw/8/GVYW5I1ui0mSBO4m2+NhesLrpYkVqOg4vWA6lLVyznE
-  GE9LzHFrAk2JOl2uE0on+boM/M97FfBSVjCNBLOycjk9L3J33U/5qpk91pp5he+D
-  mGSc2KiPx46Xga1oIR/R2FOPGPf9zcRbd0zZ3m0bpENIZt95mfmmbqGF7CPX13Ni
-  hZmmB8UCgYEAyWwPnmcCVFC/wyKdW+AsNoTDMG+XsDvxy0ddpPFpd9bauoOqo0+S
-  ZFoZDi1fbQ84x0xgTU+j7UagaOqVixBlcoJhOIadlwSxxjTDy7I5Ihf8yHhbpNWC
-  yvvtwt5QdKzkF/TziI2mhupck+s8/w/8zj8rnpxffcgBkDKt98tpRK8CgYEAyI9u
-  EFhPRyWCVEcu+jLoiDrwob8h8RhEh6+5N73RsFZtyJqltceeH8m61o8QaBTTn0Z1
-  BYUvzFeBj7RO21DxAzmNuJdsVRV+ungsSDF9Q4YF1TblHGygNtk/6iiBOTnL0GOD
-  xXq+P52wKDFAjXMZQ7BxeTYkgIgpSaQL6h5dzBcCgYBJIod5SIW7/6XhQIxKc9cZ
-  5JLK33oAb4OX/82suktFTCoXVv96VMYe0egPEcJ4YvKGyuCcsbIf0WklLyglWkq7
-  jUOntpk54GZ8RWRibc/45STP1YPREgVvlpH6E6a69pvLZxGWH3i7vBTpUJzRMAjn
-  CmDYMocpsJmAtiYctyr92QKBgAggx/UhnxTjEEUaHW1GYuYifV4QwO9kbGKTXI5F
-  wxnCZvCaU9zqGh5vroLDiYauUWVDFbsDisxdP4Lza763qQzYaEXy3REVre0gsaq7
-  VrhF1jat4tskC9otM8npmlMXZIstfUmjj9JEMb9vP457+6oinHatGw2adxHk9Bao
-  5GRBAoGBAL4FrcWE0tAUCOa3BsogsK2IYSJXLtfK6DriUG3ZcSw0GKm8XkYHhCj9
-  EdxpiU+7kFWs0aybe2nImHj+WDgbUBuCiMOIggB/Le9nfgXkr5/eh2MY0B0xocp8
-  evpyNZHj42caUrL4RzsBXwpJNOgQKliPFU3FTPQ4QEyRO4AjKq9X
-  -----END RSA PRIVATE KEY-----"

roles/store_output/tasks/main.yml

----
-- include: store_output.yml
-

roles/store_output/tasks/store_output.yml

----
-- set_fact:
-    blue_nodes_data:
-      gateways:
-      - name: "{{gwname}}"
-        address: "{{gw_ip}}"
-        bridge: "{{bridgedev}}"
-      nodes:
-      - name: plan_master
-        address: "{{plan_master_ip}}"
-      - name: luna
-        address: "{{luna_ip}}"
-      containers:
-        gateways:
-        - "{{gateway_container}}"
-        nodes:
-        - "{{plan_master_container}}"
-        - "{{px_luna_container}}"
-
-- mission_annotations: nodes='{{grey_proxy_routing.nodes|to_json}}'
-                       state='LISTENING_POST'
-  register: grey_proxy_routing_mission_annotations
-- mission_annotations: nodes='{{grey_routing.nodes|to_json}}'
-                       state={{"NOT_DEPLOYED" if deployer else "VANTAGE_POINT"}}
-                       key={{planx_key if deployer else "booger"}}
-  register: grey_routing_mission_annotations
-- mission_annotations: nodes='{{red_routing.nodes|to_json}}'
-                       state='NONE'
-  register: red_routing_mission_annotations
-  when: red is defined
-
-- set_fact:
-    grey_proxy_nodes_data:
-      gateways: "{{grey_proxy_routing.gateways}}"
-      nodes: "{{grey_proxy_routing.nodes}}"
-      networks: "{{grey_proxy_routing.networks}}"
-      top: "{{grey_proxy_routing.top}}"
-      containers:
-        gateways: "{{grey_proxy_routing.gateways|map(attribute='name')|list|index(hostvars)|map(attribute='ctr')|list}}"
-        nodes: "{{grey_proxy_routing.nodes|map(attribute='name')|list|index(hostvars)|map(attribute='node_ctr')|list}}"
-
-- set_fact:
-    grey_nodes_data:
-      gateways: "{{grey_routing.gateways}}"
-      nodes: "{{grey_routing.nodes}}"
-      networks: "{{grey_routing.networks}}"
-      top: "{{grey_routing.top}}"
-      containers:
-        gateways: "{{grey_routing.gateways|map(attribute='name')|list|index(hostvars)|map(attribute='ctr')|list}}"
-        nodes: "{{grey_routing.nodes|map(attribute='name')|list|index(hostvars)|map(attribute='node_ctr')|list}}"
-
-- set_fact:
-    red_nodes_data:
-      gateways: "{{red_routing.gateways|default([])}}"
-      nodes: "{{red_routing.nodes|default([])}}"
-      networks: "{{red_routing.networks|default([])}}"
-      top: "{{red_routing.top|default({})}}"
-      containers:
-        gateways: "{{(red_routing.gateways|default([]))|map(attribute='name')|list|index(hostvars)|map(attribute='ctr')|list}}"
-        nodes: "{{(red_routing.nodes|default([]))|map(attribute='name')|list|index(hostvars)|map(attribute='node_ctr')|list}}"
-
-- name: Write out the manifest file
-  template: src=manifest.json.j2
-            dest=./manifest.json
-
-- set_fact:
-    ipv4_records: "{{ grey_proxy_routing_mission_annotations.ipv4nets + grey_routing_mission_annotations.ipv4nets + (red_routing_mission_annotations.ipv4nets if red is defined else []) }}"
-- set_fact:
-    mission_annotations: "{{ grey_proxy_routing_mission_annotations.dannotations + grey_routing_mission_annotations.dannotations + (red_routing_mission_annotations.dannotations if red is defined else []) }}"
-- set_fact:
-    devices: "{{ grey_proxy_routing_mission_annotations.devices + grey_routing_mission_annotations.devices + (red_routing_mission_annotations.devices if red is defined else []) }}"
-- set_fact:
-    associations: "{{ grey_proxy_routing_mission_annotations.associations + grey_routing_mission_annotations.associations + (red_routing_mission_annotations.associations if red is defined else []) }}"
-- set_fact:
-    controller_annotations: "{{ grey_proxy_routing_mission_annotations.cannotations + grey_routing_mission_annotations.cannotations + (red_routing_mission_annotations.cannotations if red is defined else []) }}"
-- set_fact:
-    credentials: "{{ grey_proxy_routing_mission_annotations.credentials + grey_routing_mission_annotations.credentials + (red_routing_mission_annotations.credentials if red is defined else []) }}"
-- set_fact:
-    services: "{{ grey_proxy_routing_mission_annotations.services + grey_routing_mission_annotations.services + (red_routing_mission_annotations.services if red is defined else []) }}"
-#- set_fact:
-#    mission_annotations: "{{ mission_annotations_results | map(attribute=\"ansible_facts.annotations\") | list }}"
-
-
-- name: Write out the node file
-  template: src=nodes.json.j2
-            dest=plan/nodes.json

roles/store_output/templates/manifest.json.j2

-{
-   "blue_nodes"       : {{blue_nodes_data|to_json}},
-   "grey_proxy_nodes" : {{grey_proxy_nodes_data|to_json}},
-   "grey_nodes"       : {{grey_nodes_data|to_json}},
-   "red_nodes"        : {{red_nodes_data|to_json}}
-}

roles/store_output/templates/nodes.json.j2

-{
-  "ipv4_records" : {{ipv4_records|to_json}},
-  "device_annotations" : {{mission_annotations|to_json}},
-  "controller_annotations" : {{controller_annotations|to_json}},
-  "devices" : {{devices|to_json}},
-  "associations" : {{associations|to_json}},
-  "credentials" : {{credentials|to_json}},
-  "services" : {{services|to_json}}
-}

roles/testbed_package_install/tasks/check_for_packages.yml

----
-- stat: path=/usr/bin/pip
-  register: usr_bin_pip
-
-- check_python_module: name=docker
-  register: pkg_python_docker
-
-- stat: path=/sbin/brctl
-  register: sbin_brctl
-
-- stat: path=/usr/bin/docker
-  register: usr_bin_docker
-
-- stat: path=/usr/bin/docker.io
-  register: usr_bin_docker_io
-
-- stat: path=/usr/bin/sshpass
-  register: usr_bin_sshpass
-
-- stat: path=/usr/bin/pip2pi
-  register: usr_bin_pip2pi
-
-- set_fact:
-    system_packages:
-       pip: "{{usr_bin_pip.stat.exists}}"
-       python_docker: "{{pkg_python_docker.installed}}"
-       brctl: "{{sbin_brctl.stat.exists}}"
-       docker: "{{usr_bin_docker.stat.exists or usr_bin_docker_io.stat.exists}}"
-       sshpass: "{{usr_bin_sshpass.stat.exists}}"
-       pip2pi: "{{usr_bin_pip2pi.stat.exists}}"

roles/testbed_package_install/tasks/install_debian_packages.yml

----
-- name: Install python-pip, for pip module
-  apt: name=python-pip
-       state=present
-  when: not system_packages.pip
-- name: Install docker-py, for docker and docker_image modules
-  pip: name=docker-py
-       state=present
-  when: not system_packages.python_docker
-- name: Install bridge-utils, if necessary
-  apt: name=bridge-utils
-       state=present
-  when: not system_packages.brctl
-- name: Install docker, if necessary
-  apt: name=docker.io
-       state=present
-  when: not system_packages.docker
-- name: Install sshpass, so we can log into the support platforms
-  apt: name=sshpass
-       state=present
-  when: not system_packages.sshpass
-- name: Install pip2pi, if necessary
-  pip: name=pip2pi
-       state=present
-  when: not system_packages.pip2pi