test: strengthen molecule tests against false-passes (audit fixes)

.github/workflows/molecule.yml

@@ -105,6 +105,14 @@ jobs:
           mkdir -p ~/.ssh
           ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
 
+      - name: Acquire memory slot
+        # Coordinated admission gate. Takes an flock'd reservation on the
+        # runner host's RAM budget so the pool can scale past the point where
+        # every concurrent slot fits at peak memory_mb. Times out and proceeds
+        # without a reservation rather than failing — better one OOM than a
+        # stuck queue.
+        run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800
+
       - name: Converge
         run: molecule converge -s ${{ matrix.scenario }}
         env:
@@ -143,6 +151,13 @@ jobs:
           MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
           DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
 
+      - name: Release memory slot
+        # Runs whether converge/verify/idempotence passed or failed, so the
+        # reservation is freed promptly. Stale reservations (>1h) are also
+        # GC'd by the next acquire under the same lock as a safety net.
+        if: always()
+        run: bash scripts/wait-for-memory.sh release
+
       - name: Collect and upload diagnostics
         if: failure()
         uses: ./.github/actions/collect-diagnostics

.github/workflows/test_elasticsearch_modules.yml

@@ -17,7 +17,7 @@ on:
     paths:
       - 'roles/elasticsearch/**'
       - 'roles/elasticstack/**'
-      - 'molecule/elasticsearch_test_modules/**'
+      - 'molecule/elasticsearch_security_api/**'
       - 'molecule/shared/**'
       - '.github/workflows/test_elasticsearch_modules.yml'
       - '.github/workflows/molecule.yml'
@@ -43,7 +43,7 @@ jobs:
     needs: lint_elasticsearch
     uses: ./.github/workflows/molecule.yml
     with:
-      scenarios: '["elasticsearch_test_modules"]'
+      scenarios: '["elasticsearch_security_api"]'
       distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
       timeout: 20
     secrets: inherit

.github/workflows/test_plugins.yml

@@ -21,7 +21,7 @@ on:
     paths:
       - 'plugins/**'
       - '.github/workflows/test_plugins.yml'
-      - 'molecule/plugins/**'
+      - 'molecule/cert_info_module/**'
       - '.config/pycodestyle.cfg'
       - 'tests/**'
 
@@ -201,7 +201,7 @@ jobs:
 
       - name: Test with ansible-playbook
         run: |
-          ansible-playbook molecule/plugins/converge.yml
+          ansible-playbook molecule/cert_info_module/converge.yml
         env:
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
@@ -257,7 +257,7 @@ jobs:
 
       - name: Test with ansible-playbook
         run: |
-          ansible-playbook molecule/plugins/converge.yml
+          ansible-playbook molecule/cert_info_module/converge.yml
         env:
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
@@ -305,7 +305,7 @@ jobs:
 
       - name: Test with ansible-playbook
         run: |
-          ansible-playbook molecule/plugins/converge.yml
+          ansible-playbook molecule/cert_info_module/converge.yml
         env:
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'

.github/workflows/test_role_logstash.yml

@@ -43,6 +43,6 @@ jobs:
     needs: lint_logstash
     uses: ./.github/workflows/molecule.yml
     with:
-      scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_standalone_certs","logstash_centralized_pipelines"]'
+      scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_external_certs","logstash_standalone_certs","logstash_centralized_pipelines"]'
       distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
     secrets: inherit

molecule/beats_security/molecule.yml

@@ -11,7 +11,10 @@ platforms:
     groups:
       - elasticsearch
     distro: "${MOLECULE_DISTRO:-debian12}"
-    memory_mb: 4096
+    # ES 9.x on debian13 OOMs with 4 GB once ingest pressure starts (the
+    # new end-to-end Filebeat→ES check exercises that path). 6 GB clears
+    # it. Bumping the ES host only; Filebeat node stays at 2 GB.
+    memory_mb: 6144
   - name: "beats-sec-fb1-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}"
     groups:
       - beats

molecule/beats_security/verify.yml

@@ -86,3 +86,71 @@
               - "'certificate_authorities' in (filebeat_yml.content | b64decode)"
               - "'output.elasticsearch' in (filebeat_yml.content | b64decode)"
             fail_msg: "SSL configuration not found in filebeat.yml"
+
+        # End-to-end ingest test: emit a uniquely-tagged line via syslog on
+        # the beats node, then query ES for that exact string. Proves
+        # Filebeat actually authenticates to ES with the configured TLS
+        # client and that documents arrive — the substring checks above
+        # would all pass against a config that silently drops events
+        # (wrong CA path, wrong password, broken handshake).
+        # Write directly to the configured Filebeat input paths.
+        # `logger` goes through the syslog socket (/dev/log), which depends
+        # on rsyslog/journald-to-syslog forwarding being active; in LXC
+        # containers neither is reliably running, so the marker never
+        # lands in /var/log/syslog or /var/log/messages and Filebeat
+        # has nothing to ship. Bypass syslog by appending directly.
+        - name: Emit a unique marker line into the configured log paths
+          ansible.builtin.shell:
+            cmd: |
+              line="$(date '+%b %d %H:%M:%S') {{ ansible_facts.hostname }} molecule-beats-security[$$]: MARKER molecule-beats-security-test"
+              for f in /var/log/syslog /var/log/messages; do
+                [ -e "$f" ] && echo "$line" >> "$f"
+              done
+              # If neither file existed, create syslog so Filebeat sees it.
+              if [ ! -e /var/log/syslog ] && [ ! -e /var/log/messages ]; then
+                echo "$line" >> /var/log/syslog
+              fi
+            executable: /bin/bash
+          changed_when: false
+
+    # The ingest assertion runs on the ES host because filebeat-* lives in
+    # ES, not on the beats node.
+    - name: Confirm Filebeat shipped the marker to Elasticsearch
+      when: "'elasticsearch' in group_names"
+      block:
+        # Filebeat 8+ writes to data streams (filebeat-<version>) whose
+        # backing indices (.ds-filebeat-...) are hidden. The default
+        # expand_wildcards=open excludes them, so the search needs
+        # expand_wildcards=open,hidden, ignore_unavailable and
+        # allow_no_indices so we don't 400 if the data stream hasn't
+        # rolled over yet.
+        - name: Wait for filebeat-* index to receive the marker
+          ansible.builtin.uri:
+            url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/filebeat-*/_search?q=message:molecule-beats-security-test&expand_wildcards=open,hidden&ignore_unavailable=true&allow_no_indices=true"
+            method: GET
+            force_basic_auth: true
+            user: elastic
+            password: "{{ elastic_pass.stdout }}"
+            return_content: true
+            status_code: 200
+            validate_certs: false
+          register: _filebeat_search
+          until: (_filebeat_search.json.hits.total.value | default(0) | int) > 0
+          # Filebeat's default flush interval + ES create-index latency can
+          # exceed 5 min on slow CI runners; 60×10s = 10 min covers the slow
+          # tail while still failing fast on real breakage.
+          retries: 60
+          delay: 10
+          run_once: true  # noqa: run-once[task]
+
+        - name: Assert at least one document landed in filebeat-*
+          ansible.builtin.assert:
+            that:
+              - (_filebeat_search.json.hits.total.value | default(0) | int) > 0
+            fail_msg: >-
+              No matching document found in filebeat-*. Filebeat is running
+              and the config looks right (SSL block, output.elasticsearch
+              present), but no events made it from the beats node to ES.
+              Likely causes: wrong elastic password, wrong CA path on the
+              beats node, or TLS handshake failing silently.
+          run_once: true  # noqa: run-once[task]

molecule/cert_info_module/converge.yml

@@ -0,0 +1,107 @@
+---
+# Tests the oddly.elasticstack.cert_info module across happy and failure paths.
+# Invoked directly via `ansible-playbook` from .github/workflows/test_plugins.yml
+# (this is not a molecule scenario — there is no molecule.yml).
+- name: Converge
+  hosts: localhost
+  tasks:
+
+    # --- Happy path ---
+
+    - name: Run cert_info on a valid PKCS#12 with the right passphrase
+      oddly.elasticstack.cert_info:
+        path: files/es-ca/elastic-stack-ca.p12
+        passphrase: PleaseChangeMe
+      register: _ci_ok
+
+    - name: Show happy-path result
+      ansible.builtin.debug:
+        var: _ci_ok
+
+    - name: Assert happy-path returned the expected fields
+      ansible.builtin.assert:
+        that:
+          - _ci_ok is not failed
+          - _ci_ok.not_valid_after is defined
+          - _ci_ok.not_valid_before is defined
+          - _ci_ok.subject is defined
+          # An empty/None response would fail the date arithmetic below in
+          # surprising ways; check the date parses to a real timestamp.
+          - (_ci_ok.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')).year >= 2020
+        fail_msg: "cert_info happy-path did not return the expected dict shape: {{ _ci_ok }}"
+
+    - name: Compute days until expiry (smoke-test the typical caller pattern)
+      ansible.builtin.debug:
+        msg: >-
+          Cert expires in
+          {{ ((_ci_ok.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')) -
+              (ansible_facts.date_time.date | to_datetime('%Y-%m-%d'))).days }}
+          days
+
+    # --- Negative paths ---
+    #
+    # Each negative case must explicitly fail. The previous version used
+    # `failed_when: false` with no register/assertion, so a regression where
+    # cert_info started swallowing bad inputs (e.g. missing passphrase
+    # falling back to "" and succeeding) would have passed silently.
+
+    - name: Run cert_info with missing path argument  # noqa: args[module]
+      oddly.elasticstack.cert_info:
+        passphrase: PleaseChangeMe
+      register: _ci_missing_path
+      failed_when: false
+
+    - name: Assert missing-path call failed
+      ansible.builtin.assert:
+        that:
+          - _ci_missing_path is failed
+        fail_msg: "cert_info should have failed on missing path, but returned: {{ _ci_missing_path }}"
+
+    - name: Run cert_info with a non-existent path
+      oddly.elasticstack.cert_info:
+        path: does-not-exist.p12
+        passphrase: PleaseChangeMe
+      register: _ci_bad_path
+      failed_when: false
+
+    - name: Assert bad-path call failed
+      ansible.builtin.assert:
+        that:
+          - _ci_bad_path is failed
+        fail_msg: "cert_info should have failed on non-existent path, but returned: {{ _ci_bad_path }}"
+
+    - name: Run cert_info with the wrong passphrase
+      oddly.elasticstack.cert_info:
+        path: files/es-ca/elastic-stack-ca.p12
+        passphrase: PleaseChangeMe-wrong
+      register: _ci_bad_pass
+      failed_when: false
+
+    - name: Assert wrong-passphrase call failed
+      ansible.builtin.assert:
+        that:
+          - _ci_bad_pass is failed
+        fail_msg: "cert_info should have failed on wrong passphrase, but returned: {{ _ci_bad_pass }}"
+
+    - name: Run cert_info with no passphrase against an encrypted PKCS#12
+      oddly.elasticstack.cert_info:
+        path: files/es-ca/elastic-stack-ca.p12
+      register: _ci_no_pass
+      failed_when: false
+
+    - name: Assert no-passphrase call failed
+      ansible.builtin.assert:
+        that:
+          - _ci_no_pass is failed
+        fail_msg: "cert_info should have failed when no passphrase given for an encrypted p12, but returned: {{ _ci_no_pass }}"
+
+    - name: Run cert_info with no parameters at all  # noqa: args[module]
+      oddly.elasticstack.cert_info:
+      register: _ci_no_params
+      failed_when: false
+
+    - name: Assert no-parameters call failed
+      ansible.builtin.assert:
+        that:
+          - _ci_no_params is failed
+        fail_msg: "cert_info should have failed with no parameters, but returned: {{ _ci_no_params }}"

molecule/cert_renewal/converge.yml

@@ -114,11 +114,19 @@
     logstash_heap: "512m"
     logstash_pipeline_unsafe_shutdown: true
     kibana_tls: true
-    # Force certificate renewal
-    elasticstack_ca_will_expire_soon: true
-    elasticsearch_cert_will_expire_soon: true
-    kibana_cert_will_expire_soon: true
-    logstash_cert_will_expire_soon: true
+    # Force certificate renewal.
+    #
+    # The *_cert_will_expire_soon flags live in each role's vars/main.yml
+    # (precedence higher than play vars), so setting them here would be
+    # overridden when the role loads. Instead set the expiration buffer to
+    # a value larger than the cert's validity period — the role's
+    # cert_check_expiry task then computes "expires within buffer days" as
+    # true and flips the will_expire_soon fact via set_fact, which DOES
+    # propagate correctly to the renewal gates.
+    elasticstack_ca_expiration_buffer: 99999
+    elasticsearch_cert_expiration_buffer: 99999
+    kibana_cert_expiration_buffer: 99999
+    logstash_cert_expiration_buffer: 99999
     logstash_cert_force_regenerate: true
   tasks:
     - name: Reset shared role import guard for re-run
@@ -139,3 +147,91 @@
       ansible.builtin.include_role:
         name: oddly.elasticstack.kibana
       when: "'kibana' in group_names"
+
+# --- Phase 4: Confirm certificates actually changed ---
+#
+# Captures the post-renewal fingerprint and compares to the pre-renewal value
+# from phase 2. Facts persist across plays within a single ansible-playbook
+# run, so _*_cert_fp_before from phase 2 is still in scope here. (Verify is
+# a separate playbook invocation by molecule, which is why this comparison
+# must live in converge, not verify.)
+
+- name: Verify certificate fingerprints changed after renewal
+  hosts: all
+  vars:
+    elasticsearch_tls_key_passphrase: PleaseChangeMeIndividually
+    logstash_tls_key_passphrase: LogstashChangeMe
+    kibana_tls_key_passphrase: PleaseChangeMe
+  tasks:
+    - name: Get ES transport cert fingerprint (after renewal)
+      ansible.builtin.shell: >-
+        set -o pipefail &&
+        openssl pkcs12 -in /etc/elasticsearch/certs/{{ ansible_facts.hostname }}.p12
+        -nokeys -passin pass:{{ elasticsearch_tls_key_passphrase }} 2>/dev/null |
+        openssl x509 -fingerprint -sha256 -noout 2>/dev/null
+      args:
+        executable: /bin/bash
+      register: _es_cert_fp_after
+      changed_when: false
+      no_log: true
+      when: "'elasticsearch' in group_names"
+
+    - name: Get Logstash cert fingerprint (after renewal)
+      ansible.builtin.shell: >-
+        set -o pipefail &&
+        openssl pkcs12 -in /etc/logstash/certs/{{ ansible_facts.hostname }}-ls.p12
+        -nokeys -passin pass:{{ logstash_tls_key_passphrase }} 2>/dev/null |
+        openssl x509 -fingerprint -sha256 -noout 2>/dev/null
+      args:
+        executable: /bin/bash
+      register: _ls_cert_fp_after
+      changed_when: false
+      no_log: true
+      when: "'logstash' in group_names"
+
+    - name: Get Kibana cert fingerprint (after renewal)
+      ansible.builtin.shell: >-
+        set -o pipefail &&
+        openssl pkcs12 -in /etc/kibana/certs/{{ ansible_facts.hostname }}-kibana.p12
+        -nokeys -passin pass:{{ kibana_tls_key_passphrase }} 2>/dev/null |
+        openssl x509 -fingerprint -sha256 -noout 2>/dev/null
+      args:
+        executable: /bin/bash
+      register: _kb_cert_fp_after
+      changed_when: false
+      no_log: true
+      when: "'kibana' in group_names"
+
+    - name: Assert ES transport cert was actually replaced
+      ansible.builtin.assert:
+        that:
+          - _es_cert_fp_before.stdout != "NONE"
+          - _es_cert_fp_after.stdout | length > 0
+          - _es_cert_fp_before.stdout != _es_cert_fp_after.stdout
+        fail_msg: >-
+          ES transport cert was not renewed (before={{ _es_cert_fp_before.stdout }},
+          after={{ _es_cert_fp_after.stdout }}). The renewal play ran but the on-disk
+          p12 was not replaced — check elasticsearch_cert_will_expire_soon handling.
+      when: "'elasticsearch' in group_names"
+
+    - name: Assert Logstash cert was actually replaced
+      ansible.builtin.assert:
+        that:
+          - _ls_cert_fp_before.stdout != "NONE"
+          - _ls_cert_fp_after.stdout | length > 0
+          - _ls_cert_fp_before.stdout != _ls_cert_fp_after.stdout
+        fail_msg: >-
+          Logstash cert was not renewed (before={{ _ls_cert_fp_before.stdout }},
+          after={{ _ls_cert_fp_after.stdout }}).
+      when: "'logstash' in group_names"
+
+    - name: Assert Kibana cert was actually replaced
+      ansible.builtin.assert:
+        that:
+          - _kb_cert_fp_before.stdout != "NONE"
+          - _kb_cert_fp_after.stdout | length > 0
+          - _kb_cert_fp_before.stdout != _kb_cert_fp_after.stdout
+        fail_msg: >-
+          Kibana cert was not renewed (before={{ _kb_cert_fp_before.stdout }},
+          after={{ _kb_cert_fp_after.stdout }}).
+      when: "'kibana' in group_names"