Replace haproxy+bind with tcpproxy

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy"]
 	path = ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy
 	url = https://github.com/facebook/gnlpy.git
+[submodule "server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy"]
+	path = server/common/oursrc/scripts-proxy/vendor/inet.af/tcpproxy
+	url = https://github.com/inetaf/tcpproxy.git

ansible/roles/lvs-ldirectord/templates/ldirectord.cf.j2

@@ -40,8 +40,8 @@ virtual=2
 {% endfor %}
 	fallback=127.0.0.1 gate
 	service=http
-	request="heartbeat/http?codename=ANY"
-	virtualhost="scripts.mit.edu"
+	request="__scripts/heartbeat/http?codename=ANY"
+	virtualhost="heartbeat.scripts.scripts.mit.edu"
 	receive="1"
 	checktype=negotiate
 	checkport=80

ansible/roles/packages/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - rpm-repos

ansible/roles/proxy-scripts-proxy/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: restart scripts-proxy
+  service:
+    name: scripts-proxy
+    state: restarted
+    enabled: yes

ansible/roles/proxy-scripts-proxy/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+- name: Disable haproxy-related services
+  service:
+    name: "{{ item }}"
+    state: stopped
+    enabled: no
+  failed_when: no
+  loop:
+    - named-scripts-proxy
+    - haproxy
+- name: Remove haproxy-related configuration
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/named.scripts-proxy.conf
+    - /etc/systemd/system/named-scripts-proxy.service
+    - /etc/haproxy/haproxy.cfg
+    - /etc/rsyslog.d/haproxy.conf
+    - /etc/systemd/system/haproxy.service.d/10-scripts.conf
+    - /usr/local/bin/hatop
+- name: Remove haproxy-related packages
+  dnf:
+    name:
+      - bind
+      - bind-dlz-ldap
+      - haproxy
+    state: absent
+- name: Install scripts-proxy
+  dnf:
+    name:
+      - scripts-proxy
+    state: present
+- name: Configure scripts-proxy
+  copy:
+    dest: /etc/sysconfig/scripts-proxy
+    content: |
+      OPTIONS="-ldap_servers={{ groups['scripts-ldap'] | join(':389,') }}:389"
+  notify: restart scripts-proxy

ansible/roles/rpm-repos/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+rpm_repos:
+  - key: scripts
+    name: Scripts
+    baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}/
+    enabled: yes
+  - key: scripts-testing
+    name: Scripts Testing
+    baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}-testing/
+    enabled: "{{ enable_testing_repo | default(False) }}"

ansible/roles/rpm-repos/tasks/main.yml

@@ -0,0 +1,25 @@
+---
+- name: Configure scripts RPM repos
+  copy:
+    dest: /etc/yum.repos.d/scripts.repo
+    content: |
+      {% for repo in rpm_repos %}
+      [{{ repo.key }}]
+      name={{ repo.name }}
+      baseurl={{ repo.baseurl }}
+      enabled={{ 1 if repo.enabled else 0 }}
+      gpgcheck=0
+      {% endfor %}
+- name: Configure dnf.conf
+  ini_file:
+    path: /etc/dnf/dnf.conf
+    section: main
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+  loop:
+    - option: installonly_limit
+      value: 0
+    - option: installonlypkgs
+      value: kernel kernel-devel kernel-modules kmod-openafs ghc-cgi ghc-cgi-devel
+    - option: excludepkgs
+      value: fedora-obsolete-packages php-fpm nfs-utils

ansible/scripts-proxy.yml

@@ -13,15 +13,17 @@
       dest: /etc/munin/plugin-conf.d/
       src: files/conntrack
   roles:
+    - role: rpm-repos
+      tags: [always]
     - ansible-config-me
     - k5login
     - syslog-client
     - root-aliases
+    - mock
     - proxy-munin-node
     - nrpe
     - dnf-automatic
-    - proxy-dns
-    - proxy-haproxy
+    - proxy-scripts-proxy
     - proxy-logrotate
   tasks:
     - package:

ansible/scripts-real.yml

@@ -14,15 +14,6 @@
   vars:
     ldap_server: "{{ use_local_ldap | default(True) | ternary('ldapi://%2fvar%2frun%2fslapd-scripts.socket/', 'ldap://scripts-ldap.mit.edu/') }}"
     ldap_server_tcp: "{{ use_local_ldap | default(True) | ternary('ldap://127.0.0.1/', 'ldap://scripts-ldap.mit.edu/') }}"
-    rpm_repos:
-      - key: scripts
-        name: Scripts
-        baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}/
-        enabled: yes
-      - key: scripts-testing
-        name: Scripts Testing
-        baseurl: https://web.mit.edu/scripts/yum-repos/rpm-fc{{ ansible_distribution_major_version }}-testing/
-        enabled: "{{ enable_testing_repo | default(False) }}"
     preferred_mta: postfix
   pre_tasks:
     - name: Block Ansible on legacy realservers
@@ -39,33 +30,9 @@
         state: absent
     - include_role:
         name: real-network
-    - name: Configure dnf
-      block:
-        - name: Configure scripts RPM repos
-          copy:
-            dest: /etc/yum.repos.d/scripts.repo
-            content: |
-              {% for repo in rpm_repos %}
-              [{{ repo.key }}]
-              name={{ repo.name }}
-              baseurl={{ repo.baseurl }}
-              enabled={{ 1 if repo.enabled else 0 }}
-              gpgcheck=0
-              {% endfor %}
-        - name: Configure dnf.conf
-          ini_file:
-            path: /etc/dnf/dnf.conf
-            section: main
-            option: "{{ item.option }}"
-            value: "{{ item.value }}"
-          loop:
-            - option: installonly_limit
-              value: 0
-            - option: installonlypkgs
-              value: kernel kernel-devel kernel-modules kmod-openafs ghc-cgi ghc-cgi-devel
-            - option: excludepkgs
-              value: fedora-obsolete-packages php-fpm nfs-utils
   roles:
+    - role: rpm-repos
+      tags: [always]
     - role: packages
       tags: [always]
     - role: syslog-client

server/common/oursrc/scripts-proxy/ldap/conn.go

@@ -0,0 +1,61 @@
+package ldap
+
+import (
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	ldap "gopkg.in/ldap.v3"
+)
+
+type conn struct {
+	server, baseDn string
+	// mu protects conn during reconnect cycles
+	// TODO: The ldap package supports multiple in-flight queries;
+	// by using a Mutex we are only going to issue one at a
+	// time. We should figure out how to do retry/reconnect
+	// behavior with parallel queries.
+	mu   sync.Mutex
+	conn *ldap.Conn
+}
+
+func (c *conn) reconnect() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.conn != nil {
+		c.conn.Close()
+	}
+	var err error
+	for {
+		log.Printf("connecting to %s", c.server)
+		c.conn, err = ldap.Dial("tcp", c.server)
+		if err == nil {
+			return
+		}
+		log.Printf("connecting to %s: %v", c.server, err)
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func (c *conn) resolvePool(hostname string) (string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	escapedHostname := ldap.EscapeFilter(hostname)
+	req := &ldap.SearchRequest{
+		BaseDN:     c.baseDn,
+		Scope:      ldap.ScopeWholeSubtree,
+		Filter:     fmt.Sprintf("(|(scriptsVhostName=%s)(scriptsVhostAlias=%s))", escapedHostname, escapedHostname),
+		Attributes: []string{"scriptsVhostPoolIPv4"},
+	}
+	sr, err := c.conn.Search(req)
+	if err != nil {
+		return "", err
+	}
+	for _, entry := range sr.Entries {
+		return entry.GetAttributeValue("scriptsVhostPoolIPv4"), nil
+	}
+	// Not found is not an error
+	return "", nil
+}

server/common/oursrc/scripts-proxy/ldap/pool.go

@@ -0,0 +1,48 @@
+package ldap
+
+import "log"
+
+// Pool handles a concurrency-safe pool of connections to LDAP servers.
+type Pool struct {
+	retries int
+	// connCh holds open connections to servers.
+	connCh chan *conn
+}
+
+// NewPool constructs a connection pool that queries for baseDn from servers.
+func NewPool(servers []string, baseDn string, retries int) *Pool {
+	p := &Pool{
+		retries: retries,
+		connCh:  make(chan *conn, len(servers)),
+	}
+	for _, s := range servers {
+		c := &conn{
+			server: s,
+			baseDn: baseDn,
+		}
+		go p.reconnect(c)
+	}
+	return p
+}
+
+func (p *Pool) reconnect(c *conn) {
+	c.reconnect()
+	p.connCh <- c
+}
+
+// ResolvePool attempts to resolve the pool for hostname to an IP address, returned as a string.
+func (p *Pool) ResolvePool(hostname string) (string, error) {
+	var ip string
+	var err error
+	for i := 0; i < p.retries; i++ {
+		c := <-p.connCh
+		ip, err = c.resolvePool(hostname)
+		if err == nil {
+			p.connCh <- c
+			return ip, err
+		}
+		log.Printf("resolving %q on %s: %v", hostname, c.server, err)
+		go p.reconnect(c)
+	}
+	return ip, err
+}