diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 11fbb743..b7b40b59 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -105,6 +105,14 @@ jobs: mkdir -p ~/.ssh ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + - name: Acquire memory slot + # Coordinated admission gate. Takes an flock'd reservation on the + # runner host's RAM budget so the pool can scale past the point where + # every concurrent slot fits at peak memory_mb. Times out and proceeds + # without a reservation rather than failing — better one OOM than a + # stuck queue. + run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + - name: Converge run: molecule converge -s ${{ matrix.scenario }} env: @@ -143,6 +151,13 @@ jobs: MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + - name: Release memory slot + # Runs whether converge/verify/idempotence passed or failed, so the + # reservation is freed promptly. Stale reservations (>1h) are also + # GC'd by the next acquire under the same lock as a safety net. + if: always() + run: bash scripts/wait-for-memory.sh release + - name: Collect and upload diagnostics if: failure() uses: ./.github/actions/collect-diagnostics diff --git a/.github/workflows/test_elasticsearch_modules.yml b/.github/workflows/test_elasticsearch_modules.yml index f62fce0e..f07d70a4 100644 --- a/.github/workflows/test_elasticsearch_modules.yml +++ b/.github/workflows/test_elasticsearch_modules.yml @@ -17,7 +17,7 @@ on: paths: - 'roles/elasticsearch/**' - 'roles/elasticstack/**' - - 'molecule/elasticsearch_test_modules/**' + - 'molecule/elasticsearch_security_api/**' - 'molecule/shared/**' - '.github/workflows/test_elasticsearch_modules.yml' - '.github/workflows/molecule.yml' @@ -43,7 +43,7 @@ jobs: needs: lint_elasticsearch uses: ./.github/workflows/molecule.yml with: - scenarios: '["elasticsearch_test_modules"]' + scenarios: '["elasticsearch_security_api"]' distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }} timeout: 20 secrets: inherit diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml index ae1a8ef4..3562d0d5 100644 --- a/.github/workflows/test_plugins.yml +++ b/.github/workflows/test_plugins.yml @@ -21,7 +21,7 @@ on: paths: - 'plugins/**' - '.github/workflows/test_plugins.yml' - - 'molecule/plugins/**' + - 'molecule/cert_info_module/**' - '.config/pycodestyle.cfg' - 'tests/**' @@ -201,7 +201,7 @@ jobs: - name: Test with ansible-playbook run: | - ansible-playbook molecule/plugins/converge.yml + ansible-playbook molecule/cert_info_module/converge.yml env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' @@ -257,7 +257,7 @@ jobs: - name: Test with ansible-playbook run: | - ansible-playbook molecule/plugins/converge.yml + ansible-playbook molecule/cert_info_module/converge.yml env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' @@ -305,7 +305,7 @@ jobs: - name: Test with ansible-playbook run: | - ansible-playbook molecule/plugins/converge.yml + ansible-playbook molecule/cert_info_module/converge.yml env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index 4b779817..c434a33c 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -43,6 +43,6 @@ jobs: needs: lint_logstash uses: ./.github/workflows/molecule.yml with: - scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_standalone_certs","logstash_centralized_pipelines"]' + scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_external_certs","logstash_standalone_certs","logstash_centralized_pipelines"]' distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }} secrets: inherit diff --git a/molecule/beats_security/molecule.yml b/molecule/beats_security/molecule.yml index f60fbb55..ae0fe1cf 100644 --- a/molecule/beats_security/molecule.yml +++ b/molecule/beats_security/molecule.yml @@ -11,7 +11,10 @@ platforms: groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + # ES 9.x on debian13 OOMs with 4 GB once ingest pressure starts (the + # new end-to-end Filebeat→ES check exercises that path). 6 GB clears + # it. Bumping the ES host only; Filebeat node stays at 2 GB. + memory_mb: 6144 - name: "beats-sec-fb1-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - beats diff --git a/molecule/beats_security/verify.yml b/molecule/beats_security/verify.yml index b1160c6f..f9c2a0f4 100644 --- a/molecule/beats_security/verify.yml +++ b/molecule/beats_security/verify.yml @@ -86,3 +86,71 @@ - "'certificate_authorities' in (filebeat_yml.content | b64decode)" - "'output.elasticsearch' in (filebeat_yml.content | b64decode)" fail_msg: "SSL configuration not found in filebeat.yml" + + # End-to-end ingest test: emit a uniquely-tagged line via syslog on + # the beats node, then query ES for that exact string. Proves + # Filebeat actually authenticates to ES with the configured TLS + # client and that documents arrive — the substring checks above + # would all pass against a config that silently drops events + # (wrong CA path, wrong password, broken handshake). + # Write directly to the configured Filebeat input paths. + # `logger` goes through the syslog socket (/dev/log), which depends + # on rsyslog/journald-to-syslog forwarding being active; in LXC + # containers neither is reliably running, so the marker never + # lands in /var/log/syslog or /var/log/messages and Filebeat + # has nothing to ship. Bypass syslog by appending directly. + - name: Emit a unique marker line into the configured log paths + ansible.builtin.shell: + cmd: | + line="$(date '+%b %d %H:%M:%S') {{ ansible_facts.hostname }} molecule-beats-security[$$]: MARKER molecule-beats-security-test" + for f in /var/log/syslog /var/log/messages; do + [ -e "$f" ] && echo "$line" >> "$f" + done + # If neither file existed, create syslog so Filebeat sees it. + if [ ! -e /var/log/syslog ] && [ ! -e /var/log/messages ]; then + echo "$line" >> /var/log/syslog + fi + executable: /bin/bash + changed_when: false + + # The ingest assertion runs on the ES host because filebeat-* lives in + # ES, not on the beats node. + - name: Confirm Filebeat shipped the marker to Elasticsearch + when: "'elasticsearch' in group_names" + block: + # Filebeat 8+ writes to data streams (filebeat-) whose + # backing indices (.ds-filebeat-...) are hidden. The default + # expand_wildcards=open excludes them, so the search needs + # expand_wildcards=open,hidden, ignore_unavailable and + # allow_no_indices so we don't 400 if the data stream hasn't + # rolled over yet. + - name: Wait for filebeat-* index to receive the marker + ansible.builtin.uri: + url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/filebeat-*/_search?q=message:molecule-beats-security-test&expand_wildcards=open,hidden&ignore_unavailable=true&allow_no_indices=true" + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + return_content: true + status_code: 200 + validate_certs: false + register: _filebeat_search + until: (_filebeat_search.json.hits.total.value | default(0) | int) > 0 + # Filebeat's default flush interval + ES create-index latency can + # exceed 5 min on slow CI runners; 60×10s = 10 min covers the slow + # tail while still failing fast on real breakage. + retries: 60 + delay: 10 + run_once: true # noqa: run-once[task] + + - name: Assert at least one document landed in filebeat-* + ansible.builtin.assert: + that: + - (_filebeat_search.json.hits.total.value | default(0) | int) > 0 + fail_msg: >- + No matching document found in filebeat-*. Filebeat is running + and the config looks right (SSL block, output.elasticsearch + present), but no events made it from the beats node to ES. + Likely causes: wrong elastic password, wrong CA path on the + beats node, or TLS handshake failing silently. + run_once: true # noqa: run-once[task] diff --git a/molecule/cert_info_module/converge.yml b/molecule/cert_info_module/converge.yml new file mode 100644 index 00000000..3f05a196 --- /dev/null +++ b/molecule/cert_info_module/converge.yml @@ -0,0 +1,107 @@ +--- +# Tests the oddly.elasticstack.cert_info module across happy and failure paths. +# Invoked directly via `ansible-playbook` from .github/workflows/test_plugins.yml +# (this is not a molecule scenario — there is no molecule.yml). +- name: Converge + hosts: localhost + tasks: + + # --- Happy path --- + + - name: Run cert_info on a valid PKCS#12 with the right passphrase + oddly.elasticstack.cert_info: + path: files/es-ca/elastic-stack-ca.p12 + passphrase: PleaseChangeMe + register: _ci_ok + + - name: Show happy-path result + ansible.builtin.debug: + var: _ci_ok + + - name: Assert happy-path returned the expected fields + ansible.builtin.assert: + that: + - _ci_ok is not failed + - _ci_ok.not_valid_after is defined + - _ci_ok.not_valid_before is defined + - _ci_ok.subject is defined + # An empty/None response would fail the date arithmetic below in + # surprising ways; check the date parses to a real timestamp. + - (_ci_ok.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')).year >= 2020 + fail_msg: "cert_info happy-path did not return the expected dict shape: {{ _ci_ok }}" + + - name: Compute days until expiry (smoke-test the typical caller pattern) + ansible.builtin.debug: + msg: >- + Cert expires in + {{ ((_ci_ok.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')) - + (ansible_facts.date_time.date | to_datetime('%Y-%m-%d'))).days }} + days + + # --- Negative paths --- + # + # Each negative case must explicitly fail. The previous version used + # `failed_when: false` with no register/assertion, so a regression where + # cert_info started swallowing bad inputs (e.g. missing passphrase + # falling back to "" and succeeding) would have passed silently. + + - name: Run cert_info with missing path argument # noqa: args[module] + oddly.elasticstack.cert_info: + passphrase: PleaseChangeMe + register: _ci_missing_path + failed_when: false + + - name: Assert missing-path call failed + ansible.builtin.assert: + that: + - _ci_missing_path is failed + fail_msg: "cert_info should have failed on missing path, but returned: {{ _ci_missing_path }}" + + - name: Run cert_info with a non-existent path + oddly.elasticstack.cert_info: + path: does-not-exist.p12 + passphrase: PleaseChangeMe + register: _ci_bad_path + failed_when: false + + - name: Assert bad-path call failed + ansible.builtin.assert: + that: + - _ci_bad_path is failed + fail_msg: "cert_info should have failed on non-existent path, but returned: {{ _ci_bad_path }}" + + - name: Run cert_info with the wrong passphrase + oddly.elasticstack.cert_info: + path: files/es-ca/elastic-stack-ca.p12 + passphrase: PleaseChangeMe-wrong + register: _ci_bad_pass + failed_when: false + + - name: Assert wrong-passphrase call failed + ansible.builtin.assert: + that: + - _ci_bad_pass is failed + fail_msg: "cert_info should have failed on wrong passphrase, but returned: {{ _ci_bad_pass }}" + + - name: Run cert_info with no passphrase against an encrypted PKCS#12 + oddly.elasticstack.cert_info: + path: files/es-ca/elastic-stack-ca.p12 + register: _ci_no_pass + failed_when: false + + - name: Assert no-passphrase call failed + ansible.builtin.assert: + that: + - _ci_no_pass is failed + fail_msg: "cert_info should have failed when no passphrase given for an encrypted p12, but returned: {{ _ci_no_pass }}" + + - name: Run cert_info with no parameters at all # noqa: args[module] + oddly.elasticstack.cert_info: + register: _ci_no_params + failed_when: false + + - name: Assert no-parameters call failed + ansible.builtin.assert: + that: + - _ci_no_params is failed + fail_msg: "cert_info should have failed with no parameters, but returned: {{ _ci_no_params }}" diff --git a/molecule/plugins/files/es-ca/ca.crt b/molecule/cert_info_module/files/es-ca/ca.crt similarity index 100% rename from molecule/plugins/files/es-ca/ca.crt rename to molecule/cert_info_module/files/es-ca/ca.crt diff --git a/molecule/plugins/files/es-ca/elastic-stack-ca.p12 b/molecule/cert_info_module/files/es-ca/elastic-stack-ca.p12 similarity index 100% rename from molecule/plugins/files/es-ca/elastic-stack-ca.p12 rename to molecule/cert_info_module/files/es-ca/elastic-stack-ca.p12 diff --git a/molecule/plugins/files/es-ca/elasticsearch-cluster1.p12 b/molecule/cert_info_module/files/es-ca/elasticsearch-cluster1.p12 similarity index 100% rename from molecule/plugins/files/es-ca/elasticsearch-cluster1.p12 rename to molecule/cert_info_module/files/es-ca/elasticsearch-cluster1.p12 diff --git a/molecule/plugins/files/es-ca/elasticsearch-cluster2.p12 b/molecule/cert_info_module/files/es-ca/elasticsearch-cluster2.p12 similarity index 100% rename from molecule/plugins/files/es-ca/elasticsearch-cluster2.p12 rename to molecule/cert_info_module/files/es-ca/elasticsearch-cluster2.p12 diff --git a/molecule/cert_renewal/converge.yml b/molecule/cert_renewal/converge.yml index f094c14c..73f716dc 100644 --- a/molecule/cert_renewal/converge.yml +++ b/molecule/cert_renewal/converge.yml @@ -114,11 +114,19 @@ logstash_heap: "512m" logstash_pipeline_unsafe_shutdown: true kibana_tls: true - # Force certificate renewal - elasticstack_ca_will_expire_soon: true - elasticsearch_cert_will_expire_soon: true - kibana_cert_will_expire_soon: true - logstash_cert_will_expire_soon: true + # Force certificate renewal. + # + # The *_cert_will_expire_soon flags live in each role's vars/main.yml + # (precedence higher than play vars), so setting them here would be + # overridden when the role loads. Instead set the expiration buffer to + # a value larger than the cert's validity period — the role's + # cert_check_expiry task then computes "expires within buffer days" as + # true and flips the will_expire_soon fact via set_fact, which DOES + # propagate correctly to the renewal gates. + elasticstack_ca_expiration_buffer: 99999 + elasticsearch_cert_expiration_buffer: 99999 + kibana_cert_expiration_buffer: 99999 + logstash_cert_expiration_buffer: 99999 logstash_cert_force_regenerate: true tasks: - name: Reset shared role import guard for re-run @@ -139,3 +147,91 @@ ansible.builtin.include_role: name: oddly.elasticstack.kibana when: "'kibana' in group_names" + +# --- Phase 4: Confirm certificates actually changed --- +# +# Captures the post-renewal fingerprint and compares to the pre-renewal value +# from phase 2. Facts persist across plays within a single ansible-playbook +# run, so _*_cert_fp_before from phase 2 is still in scope here. (Verify is +# a separate playbook invocation by molecule, which is why this comparison +# must live in converge, not verify.) + +- name: Verify certificate fingerprints changed after renewal + hosts: all + vars: + elasticsearch_tls_key_passphrase: PleaseChangeMeIndividually + logstash_tls_key_passphrase: LogstashChangeMe + kibana_tls_key_passphrase: PleaseChangeMe + tasks: + - name: Get ES transport cert fingerprint (after renewal) + ansible.builtin.shell: >- + set -o pipefail && + openssl pkcs12 -in /etc/elasticsearch/certs/{{ ansible_facts.hostname }}.p12 + -nokeys -passin pass:{{ elasticsearch_tls_key_passphrase }} 2>/dev/null | + openssl x509 -fingerprint -sha256 -noout 2>/dev/null + args: + executable: /bin/bash + register: _es_cert_fp_after + changed_when: false + no_log: true + when: "'elasticsearch' in group_names" + + - name: Get Logstash cert fingerprint (after renewal) + ansible.builtin.shell: >- + set -o pipefail && + openssl pkcs12 -in /etc/logstash/certs/{{ ansible_facts.hostname }}-ls.p12 + -nokeys -passin pass:{{ logstash_tls_key_passphrase }} 2>/dev/null | + openssl x509 -fingerprint -sha256 -noout 2>/dev/null + args: + executable: /bin/bash + register: _ls_cert_fp_after + changed_when: false + no_log: true + when: "'logstash' in group_names" + + - name: Get Kibana cert fingerprint (after renewal) + ansible.builtin.shell: >- + set -o pipefail && + openssl pkcs12 -in /etc/kibana/certs/{{ ansible_facts.hostname }}-kibana.p12 + -nokeys -passin pass:{{ kibana_tls_key_passphrase }} 2>/dev/null | + openssl x509 -fingerprint -sha256 -noout 2>/dev/null + args: + executable: /bin/bash + register: _kb_cert_fp_after + changed_when: false + no_log: true + when: "'kibana' in group_names" + + - name: Assert ES transport cert was actually replaced + ansible.builtin.assert: + that: + - _es_cert_fp_before.stdout != "NONE" + - _es_cert_fp_after.stdout | length > 0 + - _es_cert_fp_before.stdout != _es_cert_fp_after.stdout + fail_msg: >- + ES transport cert was not renewed (before={{ _es_cert_fp_before.stdout }}, + after={{ _es_cert_fp_after.stdout }}). The renewal play ran but the on-disk + p12 was not replaced — check elasticsearch_cert_will_expire_soon handling. + when: "'elasticsearch' in group_names" + + - name: Assert Logstash cert was actually replaced + ansible.builtin.assert: + that: + - _ls_cert_fp_before.stdout != "NONE" + - _ls_cert_fp_after.stdout | length > 0 + - _ls_cert_fp_before.stdout != _ls_cert_fp_after.stdout + fail_msg: >- + Logstash cert was not renewed (before={{ _ls_cert_fp_before.stdout }}, + after={{ _ls_cert_fp_after.stdout }}). + when: "'logstash' in group_names" + + - name: Assert Kibana cert was actually replaced + ansible.builtin.assert: + that: + - _kb_cert_fp_before.stdout != "NONE" + - _kb_cert_fp_after.stdout | length > 0 + - _kb_cert_fp_before.stdout != _kb_cert_fp_after.stdout + fail_msg: >- + Kibana cert was not renewed (before={{ _kb_cert_fp_before.stdout }}, + after={{ _kb_cert_fp_after.stdout }}). + when: "'kibana' in group_names" diff --git a/molecule/elasticsearch_cert_content/verify.yml b/molecule/elasticsearch_cert_content/verify.yml index c291b9d6..e57a8460 100644 --- a/molecule/elasticsearch_cert_content/verify.yml +++ b/molecule/elasticsearch_cert_content/verify.yml @@ -33,33 +33,46 @@ fail_msg: "Cluster status is {{ health.json.status }}" run_once: true # noqa: run-once[task] - - name: Get HTTP certificate subject from Elasticsearch - ansible.builtin.shell: | - set -o pipefail - echo | openssl s_client -connect localhost:9200 -servername localhost 2>/dev/null | - openssl x509 -noout -subject - args: + # The whole point of the *_content variables is "bytes in == bytes on disk". + # Compare each deployed cert against the source PEM the converge fed in via + # /tmp/test-certs/ (which is what got slurped into _test_*_content facts). + # sha256sum comparison catches empty/wrong/truncated files that the + # earlier "is the subject string 'http'" heuristic let through. + - name: Hash deployed certs vs source PEM files + ansible.builtin.shell: + cmd: | + set -o pipefail + for pair in \ + "/tmp/test-certs/transport.crt /etc/elasticsearch/certs/{{ inventory_hostname }}-transport.crt" \ + "/tmp/test-certs/http.crt /etc/elasticsearch/certs/{{ inventory_hostname }}-http.crt" \ + "/tmp/test-certs/ca.crt /etc/elasticsearch/certs/ca.crt"; do + src=$(echo "$pair" | awk '{print $1}') + dst=$(echo "$pair" | awk '{print $2}') + src_sum=$(sha256sum "$src" | awk '{print $1}') + dst_sum=$(sha256sum "$dst" 2>/dev/null | awk '{print $1}' || echo "MISSING") + echo "$src $src_sum $dst $dst_sum" + done executable: /bin/bash - register: es_http_cert_subject + register: _cert_hashes changed_when: false - - name: Verify HTTP cert is the external cert (not auto-generated) - ansible.builtin.assert: - that: - - "'http' in es_http_cert_subject.stdout" - fail_msg: >- - Expected HTTP certificate subject to contain 'http' from our test cert. - Got: {{ es_http_cert_subject.stdout }} - - - name: Verify transport cert file was deployed - ansible.builtin.stat: - path: "/etc/elasticsearch/certs/{{ inventory_hostname }}-transport.crt" - register: transport_cert_stat - - - name: Assert transport cert exists + - name: Assert each deployed cert matches the source PEM byte-for-byte ansible.builtin.assert: that: - - transport_cert_stat.stat.exists + # All three lines present (transport.crt, http.crt, ca.crt). + - "(_cert_hashes.stdout_lines | length) == 3" + # No deployed file was missing. + - "(_cert_hashes.stdout_lines | select('search', ' MISSING$') | list | length) == 0" + # Each line has the form " "; + # we require src_hash == dst_hash. The regex backreference \\2 enforces it. + - >- + (_cert_hashes.stdout_lines + | map('regex_search', '^\\S+ (\\S+) \\S+ \\1$') + | select | list | length) == 3 + fail_msg: | + One or more deployed certs do not match the source PEM content the role + was fed via *_content variables. Detail (src dst src_hash dst_hash): + {{ _cert_hashes.stdout }} - name: Verify no P12 files exist (content mode is always PEM) ansible.builtin.find: @@ -73,16 +86,23 @@ - p12_files.files | length == 0 fail_msg: "Found unexpected P12 files in content mode: {{ p12_files.files | map(attribute='path') | list }}" - - name: Verify elasticsearch.yml has PEM config - ansible.builtin.command: - cmd: grep 'xpack.security.transport.ssl.certificate' /etc/elasticsearch/elasticsearch.yml - register: es_yml_check - changed_when: false + - name: Slurp elasticsearch.yml for config assertions + ansible.builtin.slurp: + src: /etc/elasticsearch/elasticsearch.yml + register: _es_yml - - name: Assert PEM-style config (not keystore) + - name: Assert verification_mode is certificate, not none, and PEM keys reference the deployed files ansible.builtin.assert: that: - - "'transport.crt' in es_yml_check.stdout" - fail_msg: >- - Expected PEM-style transport config in elasticsearch.yml. - Got: {{ es_yml_check.stdout }} + - "'xpack.security.transport.ssl.verification_mode: certificate' in (_es_yml.content | b64decode)" + - "'verification_mode: none' not in (_es_yml.content | b64decode)" + - >- + ('xpack.security.transport.ssl.certificate: certs/' ~ inventory_hostname ~ '-transport.crt') + in (_es_yml.content | b64decode) + - >- + ('xpack.security.http.ssl.certificate: certs/' ~ inventory_hostname ~ '-http.crt') + in (_es_yml.content | b64decode) + fail_msg: | + elasticsearch.yml is missing one of the expected verification_mode / cert + path lines. Content: + {{ _es_yml.content | b64decode }} diff --git a/molecule/elasticsearch_custom_certs/verify.yml b/molecule/elasticsearch_custom_certs/verify.yml index 0ba68736..ac1db841 100644 --- a/molecule/elasticsearch_custom_certs/verify.yml +++ b/molecule/elasticsearch_custom_certs/verify.yml @@ -33,23 +33,30 @@ fail_msg: "Cluster status is {{ health.json.status }}" run_once: true # noqa: run-once[task] - - name: Get HTTP certificate subject from Elasticsearch - ansible.builtin.shell: | - set -o pipefail - echo | openssl s_client -connect localhost:9200 -servername localhost 2>/dev/null | - openssl x509 -noout -subject - args: + # The previous "is the substring 'http' in the subject" heuristic only + # discriminated because the converge contrived a CN of "-http". + # Compare issuers instead: the test CA's CN is "Test CA" (set in + # converge.yml). The Elasticsearch auto-generated cert is issued by ES + # itself, so an issuer of "Test CA" proves the served cert is ours and + # the role didn't fall back to auto-generation. + - name: Get HTTP certificate issuer from Elasticsearch + ansible.builtin.shell: + cmd: | + set -o pipefail + echo | openssl s_client -connect localhost:9200 -servername localhost 2>/dev/null | + openssl x509 -noout -issuer executable: /bin/bash - register: es_http_cert_subject + register: es_http_cert_issuer changed_when: false - - name: Verify HTTP cert is the external cert (not auto-generated) + - name: Verify HTTP cert was signed by the test CA (not auto-generated) ansible.builtin.assert: that: - - "'http' in es_http_cert_subject.stdout" + - "'CN = Test CA' in es_http_cert_issuer.stdout or 'CN=Test CA' in es_http_cert_issuer.stdout" fail_msg: >- - Expected HTTP certificate subject to contain 'http' from our test cert. - Got: {{ es_http_cert_subject.stdout }} + Expected HTTP cert to be issued by the test CA "Test CA". Got: + {{ es_http_cert_issuer.stdout }}. The role may have fallen back to + its auto-generated cert path. - name: Verify transport cert file was deployed ansible.builtin.stat: diff --git a/molecule/elasticsearch_custom_certs_minimal/verify.yml b/molecule/elasticsearch_custom_certs_minimal/verify.yml index efa58f52..b1079126 100644 --- a/molecule/elasticsearch_custom_certs_minimal/verify.yml +++ b/molecule/elasticsearch_custom_certs_minimal/verify.yml @@ -69,30 +69,52 @@ - ca_cert_stat.stat.exists fail_msg: "CA cert not found — auto-extraction from PEM bundle failed" - - name: Read CA cert content - ansible.builtin.slurp: - src: /etc/elasticsearch/certs/ca.crt - register: ca_cert_content + # The point of the minimal scenario is that the role slices the CA + # out of a chained server+CA PEM bundle. A buggy role that wrote the + # full bundle to ca.crt (or even just the server cert) would still + # match 'BEGIN CERTIFICATE' — so count occurrences AND verify the + # subject matches the test CA, not the server cert. + - name: Hash extracted ca.crt and inspect its subject + ansible.builtin.shell: + cmd: | + set -o pipefail + path=/etc/elasticsearch/certs/ca.crt + count=$(grep -c '^-----BEGIN CERTIFICATE-----' "$path" || echo 0) + src_sum=$(sha256sum /tmp/test-certs/ca.crt | awk '{print $1}') + dst_sum=$(sha256sum "$path" | awk '{print $1}') + subject=$(openssl x509 -in "$path" -noout -subject) + echo "count=$count src_sum=$src_sum dst_sum=$dst_sum subject=$subject" + executable: /bin/bash + register: _ca_extract + changed_when: false - - name: Verify CA cert is not empty + - name: Assert ca.crt contains exactly one cert and matches the source CA ansible.builtin.assert: that: - - (ca_cert_content.content | b64decode) is search('BEGIN CERTIFICATE') - fail_msg: "CA cert is empty or invalid" + - "'count=1 ' in _ca_extract.stdout" + - "_ca_extract.stdout is search('CN ?= ?Test CA')" + # Source CA on /tmp/test-certs/ca.crt is the standalone CA PEM + # (single cert), so the extracted ca.crt should match it byte-for-byte. + - >- + _ca_extract.stdout is regex('src_sum=(\\S+) dst_sum=\\1') + fail_msg: | + CA cert was not extracted correctly from the chained PEM bundle. + Either the file holds the wrong number of certs, the subject is not + the test CA, or its bytes don't match the source CA. Got: + {{ _ca_extract.stdout }} - - name: Verify elasticsearch.yml uses PEM config with CA authorities - ansible.builtin.shell: | - set -o pipefail - grep 'certificate_authorities' /etc/elasticsearch/elasticsearch.yml - args: - executable: /bin/bash - register: es_yml_ca_check - changed_when: false + - name: Slurp elasticsearch.yml for cert-config assertions + ansible.builtin.slurp: + src: /etc/elasticsearch/elasticsearch.yml + register: _es_yml - - name: Assert certificate_authorities is configured + - name: Assert certificate_authorities is configured in BOTH transport and http sections ansible.builtin.assert: that: - - "'ca.crt' in es_yml_ca_check.stdout" - fail_msg: >- - Expected certificate_authorities in elasticsearch.yml. - Got: {{ es_yml_ca_check.stdout }} + - "'transport.ssl.certificate_authorities' in (_es_yml.content | b64decode)" + - "'http.ssl.certificate_authorities' in (_es_yml.content | b64decode)" + fail_msg: | + certificate_authorities missing from one of the SSL sections. The + earlier `grep ca.crt` check would have passed even if only HTTP + referenced the CA (transport gone silent). Content: + {{ _es_yml.content | b64decode }} diff --git a/molecule/elasticsearch_default/verify.yml b/molecule/elasticsearch_default/verify.yml index f31fc177..314d68bd 100644 --- a/molecule/elasticsearch_default/verify.yml +++ b/molecule/elasticsearch_default/verify.yml @@ -34,6 +34,26 @@ changed_when: false run_once: true + # The role MUST preserve the auto-generated password in initial_passwords + # when the user supplies a custom one — overwriting it (to the user's + # password) is the exact regression this block exists to catch. The + # previous version skipped the API call when _old_pass.stdout matched + # the user password, which silently turned the regression-guard into a + # no-op the moment the regression actually happened. + - name: Assert initial_passwords still holds the auto-generated password (not overwritten) # noqa: run-once[task] + ansible.builtin.assert: + that: + - _old_pass.stdout | length > 0 + - _old_pass.stdout != "TestPassword123!" + fail_msg: >- + initial_passwords was overwritten with the user-defined password. + The role is supposed to preserve the auto-generated value here + and only switch to the user password for API calls. If + initial_passwords now contains the user value, the role has + regressed and downstream services (Kibana, Logstash) that read + from this file will end up with the wrong credentials. + run_once: true + - name: Try API with old password (should fail) # noqa: run-once[task] ansible.builtin.uri: url: "https://localhost:9200/" @@ -43,7 +63,6 @@ validate_certs: false status_code: 401 run_once: true - when: _old_pass.stdout != "TestPassword123!" - name: Read cluster settings # noqa: run-once[task] ansible.builtin.uri: diff --git a/molecule/elasticsearch_diagnostics/converge.yml b/molecule/elasticsearch_diagnostics/converge.yml index 4d0461eb..b7752285 100644 --- a/molecule/elasticsearch_diagnostics/converge.yml +++ b/molecule/elasticsearch_diagnostics/converge.yml @@ -30,10 +30,15 @@ - name: Test bad config produces fast failure with diagnostics block: - - name: Inject bad setting into elasticsearch.yml + # Use a setting ES guarantees to reject (malformed YAML at the + # top level). The previous "bogus.nonexistent.setting: true" + # bait relied on ES rejecting unknown root settings, which some + # versions silently archive as "archived.bogus..." — the rescue + # below would then not fire and the test would mis-report. + - name: Inject malformed setting that ES is guaranteed to reject ansible.builtin.lineinfile: path: /etc/elasticsearch/elasticsearch.yml - line: "bogus.nonexistent.setting: true" + line: "discovery.type: bogus-invalid-discovery-type" - name: Attempt restart with bad config (should fail with diagnostics) ansible.builtin.include_role: @@ -45,13 +50,27 @@ msg: "restart_and_verify_elasticsearch.yml did not fail on bad config" rescue: - - name: Assert failure includes diagnostic log output + - name: Assert failure includes non-empty diagnostic log output ansible.builtin.assert: that: - ansible_failed_result.msg is defined - "'Recent log output' in ansible_failed_result.msg" + # The previous check only required the literal header string + # 'Recent log output' to appear — the role hardcodes that + # string, so the assertion would pass even if journalctl + # returned an empty stdout (broken diagnostics). Require + # enough trailing content that real journal output must be + # present (200+ chars beyond the header). + - ansible_failed_result.msg | length > (('Recent log output:' | length) + 200) + # And require at least one of the systemd/ES tokens that any + # real ES startup-failure log would contain. + - >- + ('elasticsearch.service' in ansible_failed_result.msg) + or ('Elasticsearch' in ansible_failed_result.msg) + or ('exit code' in ansible_failed_result.msg) fail_msg: >- - Expected failure message with 'Recent log output', got: + Expected a substantial diagnostic message containing real + journal output, got: {{ ansible_failed_result.msg | default('no message') }} - name: Restore good config diff --git a/molecule/elasticsearch_no-security/verify.yml b/molecule/elasticsearch_no-security/verify.yml index 2a2665b9..0c3f25c5 100644 --- a/molecule/elasticsearch_no-security/verify.yml +++ b/molecule/elasticsearch_no-security/verify.yml @@ -17,28 +17,13 @@ - "'xpack.security.enabled: false' in (_es_config.content | b64decode)" fail_msg: "xpack.security.enabled: false not found in elasticsearch.yml" -# Remember, this is the no-security scenario. So no https -# The comment below will create an exception for KICS security scan - - name: Health check - ansible.builtin.uri: -# kics-scan ignore-line - url: http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health - method: GET - return_content: true - status_code: 200 - validate_certs: false - register: result - until: result.json.status is defined and result.json.status in ["green", "yellow"] - retries: 6 - delay: 10 - - - name: Assert cluster is healthy - ansible.builtin.assert: - that: - - result.json.status in ["green", "yellow"] - fail_msg: "Cluster health is {{ result.json.status | default('unknown') }}" + - name: Verify cluster health (no-security, plain HTTP) + ansible.builtin.include_tasks: ../shared/verify_es_health.yml + vars: + _verify_es_scheme: http + _verify_es_status: green - - name: Node check + - name: Fetch _cat/nodes for membership check # noqa: run-once[task] ansible.builtin.uri: # kics-scan ignore-line url: http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes @@ -47,15 +32,13 @@ status_code: 200 validate_certs: false register: nodes - until: nodes.status == 200 - retries: 3 - delay: 5 + run_once: true - - name: Check if all Nodes see each other + - name: Assert every cluster member appears in _cat/nodes # noqa: run-once[task] ansible.builtin.assert: that: - - "'{{ item }}' in nodes.content" - fail_msg: "'{{ item }}' was not found in nodes.content" - success_msg: "'{{ item }}' was found in nodes.content" + - "hostvars[item].ansible_facts.hostname in nodes.content" + fail_msg: "node '{{ hostvars[item].ansible_facts.hostname }}' (host {{ item }}) was not found in _cat/nodes output" loop: "{{ groups['all'] }}" + run_once: true when: groups[elasticstack_elasticsearch_group_name] | length > 1 diff --git a/molecule/elasticsearch_test_modules/converge.yml b/molecule/elasticsearch_security_api/converge.yml similarity index 100% rename from molecule/elasticsearch_test_modules/converge.yml rename to molecule/elasticsearch_security_api/converge.yml diff --git a/molecule/elasticsearch_test_modules/create.yml b/molecule/elasticsearch_security_api/create.yml similarity index 100% rename from molecule/elasticsearch_test_modules/create.yml rename to molecule/elasticsearch_security_api/create.yml diff --git a/molecule/elasticsearch_test_modules/destroy.yml b/molecule/elasticsearch_security_api/destroy.yml similarity index 100% rename from molecule/elasticsearch_test_modules/destroy.yml rename to molecule/elasticsearch_security_api/destroy.yml diff --git a/molecule/elasticsearch_test_modules/molecule.yml b/molecule/elasticsearch_security_api/molecule.yml similarity index 100% rename from molecule/elasticsearch_test_modules/molecule.yml rename to molecule/elasticsearch_security_api/molecule.yml diff --git a/molecule/elasticsearch_test_modules/prepare.yml b/molecule/elasticsearch_security_api/prepare.yml similarity index 100% rename from molecule/elasticsearch_test_modules/prepare.yml rename to molecule/elasticsearch_security_api/prepare.yml diff --git a/molecule/elasticsearch_test_modules/requirements.yml b/molecule/elasticsearch_security_api/requirements.yml similarity index 100% rename from molecule/elasticsearch_test_modules/requirements.yml rename to molecule/elasticsearch_security_api/requirements.yml diff --git a/molecule/elasticsearch_test_modules/verify.yml b/molecule/elasticsearch_security_api/verify.yml similarity index 100% rename from molecule/elasticsearch_test_modules/verify.yml rename to molecule/elasticsearch_security_api/verify.yml diff --git a/molecule/elasticsearch_upgrade_8to9/converge.yml b/molecule/elasticsearch_upgrade_8to9/converge.yml index 956628aa..009ac262 100644 --- a/molecule/elasticsearch_upgrade_8to9/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9/converge.yml @@ -151,10 +151,64 @@ ansible.builtin.set_fact: _elasticstack_role_imported: false + # The handler-suppression fix should produce exactly one ES restart per + # node during the upgrade play: the rolling-upgrade restart itself. The + # config change introduced by elasticsearch_jvm_custom_parameters + # notifies the Restart Elasticsearch handler, which would fire a SECOND + # restart unless suppressed. Capture the start-event window so we can + # assert exactly-one after include_role completes. + - name: Capture pre-upgrade timestamp + # journalctl --since wants 'YYYY-MM-DD HH:MM:SS' (UTC implied via -u). + # Don't use --iso-8601=seconds: that emits '+00:00' which journalctl + # parses inconsistently and silently filters out every match. + ansible.builtin.command: date -u "+%Y-%m-%d %H:%M:%S" + register: _ts_before_upgrade + changed_when: false + + - name: Capture pre-upgrade ES PID + ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch' + register: _es_pid_before_upgrade + changed_when: false + - name: Include Elasticsearch role (upgrade to 9.x) ansible.builtin.include_role: name: oddly.elasticstack.elasticsearch + - name: Capture post-upgrade ES PID + ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch' + register: _es_pid_after_upgrade + changed_when: false + + - name: Count ES start events during the upgrade window + ansible.builtin.shell: + cmd: | + set -o pipefail + # systemd emits "Started Elasticsearch." with a capital E (the + # unit's Description=Elasticsearch). Use case-insensitive grep so + # we don't depend on the exact casing across distros. + journalctl -u elasticsearch \ + --since='{{ _ts_before_upgrade.stdout }}' \ + --no-pager --output=cat 2>/dev/null \ + | grep -ic 'started elasticsearch' || true + executable: /bin/bash + register: _es_start_count + changed_when: false + + - name: Assert handler-suppression worked (exactly one restart during upgrade) + ansible.builtin.assert: + that: + - _es_pid_before_upgrade.stdout != _es_pid_after_upgrade.stdout + - _es_start_count.stdout | int == 1 + fail_msg: >- + Expected exactly 1 'Started elasticsearch' event during the upgrade + play (the rolling-upgrade restart itself). Counted + {{ _es_start_count.stdout }}. If >1 the config-change handler fired + in addition to the upgrade restart — the handler-suppression fix + has regressed and ES restarted twice on this node. + success_msg: >- + Handler suppression OK: ES restarted exactly once during upgrade + (PID {{ _es_pid_before_upgrade.stdout }} → {{ _es_pid_after_upgrade.stdout }}). + # Re-run with elasticstack_version: "latest" to exercise the post-install # detection path. Since we're already on the latest 9.x, the package should # NOT change and the rolling restart should NOT trigger. This verifies that diff --git a/molecule/elasticsearch_upgrade_8to9/verify.yml b/molecule/elasticsearch_upgrade_8to9/verify.yml index 329da2cd..a6a2c5bf 100644 --- a/molecule/elasticsearch_upgrade_8to9/verify.yml +++ b/molecule/elasticsearch_upgrade_8to9/verify.yml @@ -74,18 +74,19 @@ success_msg: "Cluster is {{ cluster_health.json.status }} with {{ cluster_health.json.number_of_nodes }} nodes" - name: Check all nodes are visible + # Default _cat/nodes columns don't include version — must specify h= + # explicitly. Without it the per-node version assert below fails + # with 'dict object has no attribute "version"'. ansible.builtin.uri: - url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?format=json + url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name,version&format=json method: GET force_basic_auth: true user: elastic password: "{{ elastic_pass.stdout }}" return_content: true + status_code: 200 validate_certs: false register: nodes_info - until: nodes_info.status | default(0) == 200 - retries: 3 - delay: 5 when: groups[elasticstack_elasticsearch_group_name] | length > 1 - name: Display nodes @@ -93,6 +94,19 @@ var: nodes_info.json when: groups[elasticstack_elasticsearch_group_name] | length > 1 + - name: Assert every cluster node is on 9.x (not just the queried one) + ansible.builtin.assert: + that: + - item.version is version('9.0.0', '>=') + - item.version is version('10.0.0', '<') + fail_msg: >- + Node {{ item.name }} reports version {{ item.version }} — expected 9.x. + The rolling upgrade left at least one node on a stale version. + loop: "{{ nodes_info.json }}" + loop_control: + label: "{{ item.name }} ({{ item.version }})" + when: groups[elasticstack_elasticsearch_group_name] | length > 1 + - name: Verify test data survived the upgrade # noqa: run-once[task] ansible.builtin.uri: url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/upgrade-test-index/_doc/1 @@ -120,18 +134,35 @@ success_msg: "Test data survived upgrade: {{ test_doc.json._source.test_field }}" run_once: true - - name: Verify JVM custom config was written during upgrade - ansible.builtin.slurp: - src: /etc/elasticsearch/jvm.options.d/90-custom.options - register: custom_jvm_config + - name: Query running JVM input arguments + ansible.builtin.uri: + url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_nodes/jvm?filter_path=nodes.*.jvm.input_arguments + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + return_content: true + status_code: 200 + validate_certs: false + register: _nodes_jvm - - name: Check custom JVM parameter is on disk (written pre-upgrade, applied on upgrade restart) + - name: Assert custom JVM parameter is loaded by the running ES process (not just on disk) ansible.builtin.assert: that: - - "'-Des.upgrade.test.marker=true' in (custom_jvm_config.content | b64decode)" + - >- + '-Des.upgrade.test.marker=true' in ( + _nodes_jvm.json.nodes.values() + | map(attribute='jvm') + | map(attribute='input_arguments') + | flatten + ) fail_msg: >- - JVM custom config not applied. Got: {{ custom_jvm_config.content | b64decode }} - success_msg: "JVM custom parameter correctly applied during upgrade" + The role wrote /etc/elasticsearch/jvm.options.d/90-custom.options with + -Des.upgrade.test.marker=true, but it is NOT in the running JVM's + input_arguments. The upgrade restart did not pick up the new options + file — either the file was written after the restart, or the upgrade + path skipped the restart entirely. + success_msg: "Custom JVM parameter is in the running JVM — upgrade restart picked up the new options" - name: Final upgrade verification summary # noqa: run-once[task] ansible.builtin.debug: diff --git a/molecule/elasticsearch_upgrade_8to9_single/converge.yml b/molecule/elasticsearch_upgrade_8to9_single/converge.yml index 9eba0fe1..2960019b 100644 --- a/molecule/elasticsearch_upgrade_8to9_single/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9_single/converge.yml @@ -124,6 +124,25 @@ ansible.builtin.include_role: name: oddly.elasticstack.repos + # The role's elasticstack_version: latest path does NOT trigger the + # rolling-upgrade detection on same-major bumps — only pinned-higher + # version OR release change qualifies (see + # roles/elasticsearch/tasks/elasticsearch-upgrade-detection.yml). + # And the chiark apt/yum mirror's cached metadata may not surface + # newer 8.x patches without a forced refresh. Drive the package + + # restart directly here, as the original test did. Worth a follow-up + # to add a "latest-mode same-major upgrade detection" path to the + # role, but that's a role change, not a test change. + - name: Force apt cache update before 8.x upgrade + ansible.builtin.apt: + update_cache: true + when: ansible_facts.os_family == 'Debian' + + - name: Force dnf metadata refresh before 8.x upgrade + ansible.builtin.command: dnf clean expire-cache + changed_when: false + when: ansible_facts.os_family == 'RedHat' + - name: Upgrade to latest 8.x # noqa: package-latest ansible.builtin.package: name: elasticsearch @@ -147,6 +166,27 @@ delay: 10 until: health.status == 200 + - name: Confirm test index survived the 8.x→8.x upgrade + ansible.builtin.uri: + url: https://localhost:9200/upgrade-test/_doc/1 + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + return_content: true + validate_certs: false + status_code: 200 + register: _post_8x_doc + + - name: Assert test data survived 8.x→8.x upgrade (catches data loss before 9.x step) + ansible.builtin.assert: + that: + - _post_8x_doc.json._source.message == "pre-upgrade-test-data" + fail_msg: >- + Test data was lost during the 8.x→8.x upgrade. A subsequent + failure in the 9.x verify would otherwise be blamed on the + 9.x step. + - name: Verify latest 8.x before 9.x upgrade hosts: all vars: diff --git a/molecule/elasticsearch_upgrade_8to9_single/verify.yml b/molecule/elasticsearch_upgrade_8to9_single/verify.yml index dc307803..6b675e38 100644 --- a/molecule/elasticsearch_upgrade_8to9_single/verify.yml +++ b/molecule/elasticsearch_upgrade_8to9_single/verify.yml @@ -31,6 +31,7 @@ ansible.builtin.assert: that: - es_info.json.version.number is version('9.0.0', '>=') + - es_info.json.version.number is version('10.0.0', '<') fail_msg: "Expected ES 9.x but got {{ es_info.json.version.number }}" success_msg: "ES 9.x ({{ es_info.json.version.number }}) - upgrade successful!" diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index ebe980ea..0d44267e 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -77,6 +77,50 @@ msg: "Logstash events — in: {{ logstash_stats.json.pipelines.main.events.in }}, out: {{ logstash_stats.json.pipelines.main.events.out }}" when: "'logstash' in group_names" + # End-to-end: Beats → Logstash is proven by events.out > 0 above. To + # prove Logstash → ES (the gap a config-only verify silently misses), + # query ES for any non-system index with documents. The flagship scenario + # has Filebeat+Auditbeat+Metricbeat shipping to Logstash, which outputs + # to ES with the default `logstash-*` / data-stream `logs-*` naming — + # either appears here. + # Use _count instead of _cat/indices because selectattr with a literal + # dot in the attribute name ('docs.count') is parsed by Jinja as a + # nested attribute access (item.docs.count) and silently never matches. + # _count returns {"count": N} which is simple to assert on. + - name: Confirm Logstash output index landed in Elasticsearch + ansible.builtin.uri: + url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/logstash-*,logs-*,ecs-*/_count?expand_wildcards=open,hidden&ignore_unavailable=true&allow_no_indices=true" + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + return_content: true + status_code: 200 + validate_certs: false + register: _ls_es_count + until: (_ls_es_count.json.count | default(0) | int) > 0 + # Generous retry budget: in CI the Beats → Logstash → ES chain needs + # cluster green + Logstash plugin warmup + ES create-index latency. + retries: 60 + delay: 10 + delegate_to: "{{ elasticsearch_ca }}" + run_once: true + when: "'logstash' in group_names" # noqa: run-once[task] + + - name: Assert at least one Logstash output document exists in ES + ansible.builtin.assert: + that: + - (_ls_es_count.json.count | default(0) | int) > 0 + fail_msg: >- + Logstash reported events.out > 0 to its internal counter, but no + documents landed in ES under logstash-*, logs-*, or ecs-* indices. + The Beats → Logstash leg works but Logstash → ES does not — likely + a wrong elastic password in the ES output, a TLS chain failure to + ES, or the output plugin silently dropping events. + run_once: true + delegate_to: "{{ elasticsearch_ca }}" + when: "'logstash' in group_names" # noqa: run-once[task] + - name: Run Kibana checks when: "'kibana' in group_names" block: diff --git a/molecule/kibana_custom/verify.yml b/molecule/kibana_custom/verify.yml index de04ef16..b5aca810 100644 --- a/molecule/kibana_custom/verify.yml +++ b/molecule/kibana_custom/verify.yml @@ -79,6 +79,21 @@ fail_msg: "Sniff on start not found in kibana.yml" when: (lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int) < 9 + # Negative test on 9.x: even with kibana_sniff_on_start: true the + # template should NOT emit sniffOnStart (Kibana 9.x removed the + # sniffing setting). Without this assert, a regression that restored + # the line on 9.x would only show up under default-release CI as a + # confusing Kibana startup warning, not a test failure. + - name: Verify sniff on start is NOT emitted on 9.x + ansible.builtin.assert: + that: + - "'sniffOnStart' not in (kibana_yml.content | b64decode)" + fail_msg: >- + kibana_sniff_on_start: true should be silently dropped on + Kibana 9.x, but sniffOnStart appears in kibana.yml. Kibana + 9.x will log a warning at startup and ignore the value. + when: (lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int) >= 9 + - name: Check for kibana.yml backup files ansible.builtin.find: paths: /etc/kibana diff --git a/molecule/kibana_custom_certs/verify.yml b/molecule/kibana_custom_certs/verify.yml index 45c639fc..ac051b9c 100644 --- a/molecule/kibana_custom_certs/verify.yml +++ b/molecule/kibana_custom_certs/verify.yml @@ -77,3 +77,39 @@ that: - "'server.ssl.enabled: true' in (kibana_yml.content | b64decode)" fail_msg: "server.ssl.enabled: true should be present when kibana_tls is true" + + # An open https://localhost:5601 with validate_certs:false doesn't + # prove the role rendered the right cert paths — Kibana could be + # serving its bundled default cert if the role's PEM branch picked + # up the wrong filename (e.g. wrong _kibana_cert_format default + # silently re-routed to the .p12 keystore.path branch). + - name: Verify server.ssl.certificate and server.ssl.key paths are rendered + ansible.builtin.assert: + that: + - "'server.ssl.certificate:' in (kibana_yml.content | b64decode)" + - "'server.ssl.key:' in (kibana_yml.content | b64decode)" + - "'keystore.path' not in (kibana_yml.content | b64decode)" + - "'elasticsearch.ssl.verificationMode: none' in (kibana_yml.content | b64decode)" + fail_msg: | + kibana.yml is missing PEM cert/key paths, or accidentally rendered + a keystore.path entry (wrong cert format detection), or dropped + the elasticsearch.ssl.verificationMode: none line from extra_config. + Got: + {{ kibana_yml.content | b64decode }} + + # An open socket on 5601 doesn't prove the cert chains to our test CA. + # Confirm the served cert is signed by "Test CA" — the role's + # auto-generated fallback would have a different issuer. + - name: TLS handshake against Kibana 5601 (signed by Test CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5601 \ + -CAfile /tmp/test-certs/ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' + executable: /bin/bash + register: _kb_tls_probe + changed_when: false diff --git a/molecule/logstash_centralized_pipelines/verify.yml b/molecule/logstash_centralized_pipelines/verify.yml index ada80fa7..9fedd35d 100644 --- a/molecule/logstash_centralized_pipelines/verify.yml +++ b/molecule/logstash_centralized_pipelines/verify.yml @@ -108,8 +108,33 @@ path: /etc/logstash/conf.d/main register: pipeline_dir - - name: Warn if local pipeline configs exist - ansible.builtin.debug: - msg: "Local pipeline configs should not exist with centralized management" - when: pipeline_dir.stat.exists and (not pipeline_dir.stat.isdir or pipeline_dir.stat.size > 0) - changed_when: true + - name: Assert no local pipeline configs exist with centralized management + ansible.builtin.assert: + that: + - >- + (not pipeline_dir.stat.exists) or + (pipeline_dir.stat.isdir and pipeline_dir.stat.size == 0) + fail_msg: >- + /etc/logstash/conf.d/main exists with content despite + logstash_no_pipelines: true. The role should not have written + local pipeline files when xpack.management is configured. + + # --- Confirm Logstash actually starts --- + # + # The previous version of this verify only inspected /etc/logstash/logstash.yml + # — never confirmed Logstash came up. If xpack.management.* was misconfigured + # (malformed YAML, missing required field) Logstash crashes on startup and + # every assertion above still passes because they only test rendered config. + # + # We can ONLY check the service is active here. The scenario's ES hosts + # (es-001.example.com, es-002.example.com) are intentionally fake to test + # config rendering without a real cluster, so the Logstash monitoring API + # may never bind and the log will contain xpack.management connection + # errors — those are EXPECTED here and not a regression signal. + - name: Check Logstash service is running + ansible.builtin.service: + name: logstash + state: started + check_mode: true + register: _ls_svc + failed_when: _ls_svc.changed diff --git a/molecule/logstash_external_certs/converge.yml b/molecule/logstash_external_certs/converge.yml new file mode 100644 index 00000000..00892855 --- /dev/null +++ b/molecule/logstash_external_certs/converge.yml @@ -0,0 +1,38 @@ +--- +# Tests Logstash with beats input SSL using external certificates. +# This validates the logstash_cert_source: external path with beats input +# (as opposed to logstash_ssl which tests elastic_agent input). +- name: Deploy Logstash with beats SSL using external certs + hosts: all + vars: + elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int }}" + elasticstack_full_stack: false + elasticstack_no_log: false + logstash_output_elasticsearch: false + logstash_create_user: false + logstash_create_role: false + logstash_pipeline_unsafe_shutdown: true + logstash_heap: "512m" + # Beats input with SSL enabled + logstash_input_beats: true + logstash_input_beats_ssl: true + logstash_input_beats_port: 5044 + # External certificate configuration + logstash_cert_source: external + logstash_tls_certificate_file: /tmp/test-certs/server.crt + logstash_tls_key_file: /tmp/test-certs/server.key + logstash_tls_ca_file: /tmp/test-certs/ca.crt + logstash_tls_remote_src: true + logstash_extra_outputs: | + stdout { codec => rubydebug } + tasks: + - name: Generate test certificates + ansible.builtin.include_tasks: ../shared/generate_test_certs.yml + + - name: Include Elastic repos role + ansible.builtin.include_role: + name: oddly.elasticstack.repos + + - name: Include Logstash role + ansible.builtin.include_role: + name: oddly.elasticstack.logstash diff --git a/molecule/logstash_external_certs/create.yml b/molecule/logstash_external_certs/create.yml new file mode 120000 index 00000000..138c2c2f --- /dev/null +++ b/molecule/logstash_external_certs/create.yml @@ -0,0 +1 @@ +../shared/create.yml \ No newline at end of file diff --git a/molecule/logstash_external_certs/destroy.yml b/molecule/logstash_external_certs/destroy.yml new file mode 120000 index 00000000..d18e9a34 --- /dev/null +++ b/molecule/logstash_external_certs/destroy.yml @@ -0,0 +1 @@ +../shared/destroy.yml \ No newline at end of file diff --git a/molecule/logstash_external_certs/molecule.yml b/molecule/logstash_external_certs/molecule.yml new file mode 100644 index 00000000..6d5d1496 --- /dev/null +++ b/molecule/logstash_external_certs/molecule.yml @@ -0,0 +1,28 @@ +--- +prerun: false +dependency: + name: galaxy + options: + requirements-file: requirements.yml +driver: + name: default +platforms: + - name: "ls-certs-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" + groups: + - logstash + distro: "${MOLECULE_DISTRO:-debian12}" + memory_mb: 2048 +provisioner: + name: ansible + env: + ANSIBLE_LOG_PATH: /var/log/ansible.log + connection_options: + ansible_connection: ssh + ansible_user: root + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + inventory: + group_vars: + all: + ansible_python_interpreter: /usr/bin/python3 +verifier: + name: ansible diff --git a/molecule/logstash_external_certs/requirements.yml b/molecule/logstash_external_certs/requirements.yml new file mode 100644 index 00000000..554b93f4 --- /dev/null +++ b/molecule/logstash_external_certs/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - community.general + - community.crypto diff --git a/molecule/logstash_external_certs/verify.yml b/molecule/logstash_external_certs/verify.yml new file mode 100644 index 00000000..c1253132 --- /dev/null +++ b/molecule/logstash_external_certs/verify.yml @@ -0,0 +1,161 @@ +--- +- name: Verify Logstash with beats SSL and external certs + hosts: all + tasks: + - name: Check Logstash service is running + ansible.builtin.service: + name: logstash + state: started + check_mode: true + register: logstash_service + failed_when: logstash_service.changed + + # --- Certificate verification --- + + - name: Check certificate directory exists + ansible.builtin.stat: + path: /etc/logstash/certs + register: certs_dir + + - name: Assert certificate directory exists + ansible.builtin.assert: + that: + - certs_dir.stat.exists + - certs_dir.stat.isdir + fail_msg: "Certificate directory /etc/logstash/certs not found" + + - name: Check server certificate was copied + ansible.builtin.stat: + path: "/etc/logstash/certs/{{ ansible_facts.hostname }}-server.crt" + register: server_cert + + - name: Assert server certificate exists + ansible.builtin.assert: + that: + - server_cert.stat.exists + fail_msg: "Server certificate not found in /etc/logstash/certs/" + + - name: Check server key was copied + ansible.builtin.stat: + path: "/etc/logstash/certs/{{ ansible_facts.hostname }}.key" + register: server_key + + - name: Assert server key exists + ansible.builtin.assert: + that: + - server_key.stat.exists + fail_msg: "Server key not found in /etc/logstash/certs/" + + - name: Verify server key is unencrypted PKCS#8 PEM + ansible.builtin.command: head -1 /etc/logstash/certs/{{ ansible_facts.hostname }}.key + register: _server_key_header + changed_when: false + + - name: Assert PKCS#8 header + ansible.builtin.assert: + that: + - _server_key_header.stdout == '-----BEGIN PRIVATE KEY-----' + fail_msg: >- + Expected unencrypted PKCS#8 PEM (-----BEGIN PRIVATE KEY-----), + got {{ _server_key_header.stdout }}. + + - name: Check CA certificate was copied + ansible.builtin.stat: + path: /etc/logstash/certs/ca.crt + register: ca_cert + + - name: Assert CA certificate exists + ansible.builtin.assert: + that: + - ca_cert.stat.exists + fail_msg: "CA certificate not found in /etc/logstash/certs/" + + # --- Input SSL verification --- + + - name: Read input configuration + ansible.builtin.slurp: + src: /etc/logstash/conf.d/main/10-input.conf + register: input_content + + - name: Verify beats input with SSL is configured + ansible.builtin.assert: + that: + - "'beats {' in (input_content.content | b64decode)" + - "'port => 5044' in (input_content.content | b64decode)" + - "'ssl_enabled' in (input_content.content | b64decode) or 'ssl => true' in (input_content.content | b64decode)" + fail_msg: "Beats input with SSL not properly configured" + + # --- Port check --- + + - name: Check Logstash is listening on port 5044 + ansible.builtin.wait_for: + port: 5044 + timeout: 120 + state: started + register: port_check + ignore_errors: true + + # An open socket doesn't prove TLS — Logstash could be serving plain text + # on 5044 if the SSL block silently failed to load. Run an actual TLS + # handshake against the port and verify the served cert chains to the + # external CA we deployed. + - name: TLS handshake against Logstash 5044 (signed by Test CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5044 \ + -CAfile /tmp/test-certs/ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' + executable: /bin/bash + register: _ls_tls_probe + changed_when: false + when: port_check is not failed + + - name: Show diagnostics on port check failure + when: port_check is failed + block: + - name: Show Logstash log after failure + ansible.builtin.shell: >- + journalctl -u logstash --no-pager -n 100 2>/dev/null || + tail -100 /var/log/logstash/logstash-plain.log 2>/dev/null || + echo "No logs found" + register: logstash_log + changed_when: false + + - name: Display failure log # noqa: no-handler + ansible.builtin.debug: + var: logstash_log.stdout_lines + + - name: Fail with context + ansible.builtin.fail: + msg: "Logstash port 5044 did not open within timeout." + + # --- Config syntax validation --- + + - name: Get installed Logstash version + ansible.builtin.shell: >- + set -o pipefail && + (v=$(dpkg-query -W -f='${Version}' logstash 2>/dev/null) && + echo "${v}" | sed 's/^[0-9]*://;s/-.*$//') || + rpm -q --qf '%{VERSION}' logstash 2>/dev/null + args: + executable: /bin/bash + register: logstash_version + changed_when: false + + - name: Verify Logstash config syntax + ansible.builtin.command: >- + /usr/share/logstash/bin/logstash + --path.settings=/etc/logstash + --config.test_and_exit + -f /etc/logstash/conf.d/main/ + become: "{{ logstash_version.stdout is version('9.0.0', '>=', version_type='loose') }}" + become_method: ansible.builtin.su + become_user: "{{ 'logstash' if logstash_version.stdout is version('9.0.0', '>=', version_type='loose') else omit }}" + become_flags: '-s /bin/sh' + register: config_test + changed_when: false + failed_when: config_test.rc != 0 diff --git a/molecule/logstash_ssl/verify.yml b/molecule/logstash_ssl/verify.yml index 2c9eff06..945c3dda 100644 --- a/molecule/logstash_ssl/verify.yml +++ b/molecule/logstash_ssl/verify.yml @@ -103,6 +103,25 @@ register: port_check ignore_errors: true + # An open socket doesn't prove TLS — Logstash could be serving plain text + # on 5044 if the SSL block silently failed to load. Run an actual TLS + # handshake against the port and verify the served cert chains to our + # test CA. + - name: TLS handshake against Logstash 5044 (signed by Test CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5044 \ + -CAfile /tmp/test-certs/ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' + executable: /bin/bash + register: _ls_tls_probe + changed_when: false + when: port_check is not failed + - name: Show diagnostics on port check failure when: port_check is failed block: diff --git a/molecule/logstash_standalone_certs/converge.yml b/molecule/logstash_standalone_certs/converge.yml index 00892855..d8f10816 100644 --- a/molecule/logstash_standalone_certs/converge.yml +++ b/molecule/logstash_standalone_certs/converge.yml @@ -1,8 +1,9 @@ --- -# Tests Logstash with beats input SSL using external certificates. -# This validates the logstash_cert_source: external path with beats input -# (as opposed to logstash_ssl which tests elastic_agent input). -- name: Deploy Logstash with beats SSL using external certs +# Exercises logstash_cert_source: standalone — the path where Logstash +# generates its own self-signed CA + server cert and PKCS12 keystore. +# Previously uncovered by any molecule scenario (the directory named +# logstash_standalone_certs actually tested cert_source: external). +- name: Converge hosts: all vars: elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int }}" @@ -13,22 +14,14 @@ logstash_create_role: false logstash_pipeline_unsafe_shutdown: true logstash_heap: "512m" - # Beats input with SSL enabled logstash_input_beats: true logstash_input_beats_ssl: true logstash_input_beats_port: 5044 - # External certificate configuration - logstash_cert_source: external - logstash_tls_certificate_file: /tmp/test-certs/server.crt - logstash_tls_key_file: /tmp/test-certs/server.key - logstash_tls_ca_file: /tmp/test-certs/ca.crt - logstash_tls_remote_src: true + logstash_cert_source: standalone + logstash_tls_key_passphrase: StandaloneTestPass logstash_extra_outputs: | stdout { codec => rubydebug } tasks: - - name: Generate test certificates - ansible.builtin.include_tasks: ../shared/generate_test_certs.yml - - name: Include Elastic repos role ansible.builtin.include_role: name: oddly.elasticstack.repos diff --git a/molecule/logstash_standalone_certs/molecule.yml b/molecule/logstash_standalone_certs/molecule.yml index 6d5d1496..4f4abd03 100644 --- a/molecule/logstash_standalone_certs/molecule.yml +++ b/molecule/logstash_standalone_certs/molecule.yml @@ -7,7 +7,7 @@ dependency: driver: name: default platforms: - - name: "ls-certs-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" + - name: "ls-standalone-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" diff --git a/molecule/logstash_standalone_certs/prepare.yml b/molecule/logstash_standalone_certs/prepare.yml new file mode 100644 index 00000000..4684fe4e --- /dev/null +++ b/molecule/logstash_standalone_certs/prepare.yml @@ -0,0 +1,68 @@ +--- +- name: Prepare + hosts: all + tasks: + - name: Populate /etc/hosts with molecule instances + ansible.builtin.lineinfile: + path: /etc/hosts + regexp: ".*{{ item }}$" + line: "{{ hostvars[item]['ansible_host'] }} {{ item }}" + loop: "{{ groups['all'] }}" + when: groups['all'] | length > 1 + + - name: Common prepare tasks + ansible.builtin.include_tasks: ../shared/prepare_common.yml + + # The standalone cert_source path in roles/logstash still distributes + # a CA file from elasticstack_ca_host/elasticstack_ca_dir to the + # Logstash node (line 434 of logstash-security.yml). This scenario + # has no ES, so we fake the CA host: elasticstack_ca_host defaults + # to inventory_hostname when no elasticsearch group exists, so + # pre-seed /opt/es-ca/ca.crt on the Logstash node with a placeholder + # cert that cert_distribute can fetch from itself. + - name: Create faux ES CA directory + ansible.builtin.file: + path: /opt/es-ca + state: directory + owner: root + group: root + mode: "0700" + + - name: Generate placeholder ES CA key + ansible.builtin.command: + cmd: openssl genrsa -out /opt/es-ca/ca.key 2048 + creates: /opt/es-ca/ca.key + + - name: Generate placeholder ES CA cert + ansible.builtin.command: + cmd: >- + openssl req -x509 -new -nodes + -key /opt/es-ca/ca.key + -sha256 -days 365 + -subj "/CN=Faux ES CA for standalone test" + -out /opt/es-ca/ca.crt + creates: /opt/es-ca/ca.crt + + # logstash-security.yml's "Manage Elasticsearch user and role" block runs + # unconditionally for cert_mode == 'standalone' and calls fetch_password + # against elasticstack_initial_passwords (default /usr/share/elasticsearch/ + # initial_passwords). With no ES installed, the file doesn't exist and + # the role fails before reaching the actual user/role creation tasks + # (which we suppress via logstash_create_user/role: false in converge). + # Seed the file with a dummy elastic password so the fetch succeeds. + - name: Create faux ES install directory + ansible.builtin.file: + path: /usr/share/elasticsearch + state: directory + owner: root + group: root + mode: "0755" + + - name: Seed faux initial_passwords for fetch_password.yml + ansible.builtin.copy: + dest: /usr/share/elasticsearch/initial_passwords + content: | + PASSWORD elastic = dummy-password-for-standalone-test + owner: root + group: root + mode: "0600" diff --git a/molecule/logstash_standalone_certs/verify.yml b/molecule/logstash_standalone_certs/verify.yml index 12bc4276..9c1ed8ea 100644 --- a/molecule/logstash_standalone_certs/verify.yml +++ b/molecule/logstash_standalone_certs/verify.yml @@ -1,5 +1,5 @@ --- -- name: Verify Logstash with beats SSL and external certs +- name: Verify Logstash standalone certs hosts: all tasks: - name: Check Logstash service is running @@ -7,136 +7,155 @@ name: logstash state: started check_mode: true - register: logstash_service - failed_when: logstash_service.changed + register: _ls_svc + failed_when: _ls_svc.changed - # --- Certificate verification --- + # --- Standalone-generated files exist --- - - name: Check certificate directory exists + - name: Stat each standalone artefact ansible.builtin.stat: - path: /etc/logstash/certs - register: certs_dir - - - name: Assert certificate directory exists + path: "{{ item }}" + register: _ls_artefacts + loop: + - /etc/logstash/certs/standalone-ca.crt + - /etc/logstash/certs/standalone-ca.key + - /etc/logstash/certs/{{ inventory_hostname }}-server.crt + - /etc/logstash/certs/{{ inventory_hostname }}.key + - /etc/logstash/certs/keystore.pfx + - /etc/logstash/certs/ca.crt + + - name: Assert every standalone-generated cert/key/keystore exists ansible.builtin.assert: that: - - certs_dir.stat.exists - - certs_dir.stat.isdir - fail_msg: "Certificate directory /etc/logstash/certs not found" - - - name: Check server certificate was copied - ansible.builtin.stat: - path: "/etc/logstash/certs/{{ ansible_facts.hostname }}-server.crt" - register: server_cert + - item.stat.exists + fail_msg: "{{ item.item }} not generated by standalone cert_source path" + loop: "{{ _ls_artefacts.results }}" + loop_control: + label: "{{ item.item }}" - - name: Assert server certificate exists + - name: Assert standalone cert permissions are 0640 ansible.builtin.assert: that: - - server_cert.stat.exists - fail_msg: "Server certificate not found in /etc/logstash/certs/" + - item.stat.mode == "0640" + fail_msg: "{{ item.item }} has mode {{ item.stat.mode }}, expected 0640" + loop: "{{ _ls_artefacts.results }}" + loop_control: + label: "{{ item.item }}" + + # --- Standalone CA subject matches the hardcoded role value --- + + - name: Inspect standalone CA subject + ansible.builtin.command: + cmd: openssl x509 -in /etc/logstash/certs/standalone-ca.crt -noout -subject + register: _ca_subject + changed_when: false - - name: Check server key was copied - ansible.builtin.stat: - path: "/etc/logstash/certs/{{ ansible_facts.hostname }}.key" - register: server_key + - name: Assert standalone CA was self-signed with the expected CN + ansible.builtin.assert: + that: + - "'CN = Logstash Standalone CA' in _ca_subject.stdout or 'CN=Logstash Standalone CA' in _ca_subject.stdout" + fail_msg: | + standalone-ca.crt subject is unexpected. The role hardcodes + /CN=Logstash Standalone CA. Got: {{ _ca_subject.stdout }} + + # --- Server cert chains to the standalone CA --- + + - name: Verify server cert was signed by the standalone CA + ansible.builtin.shell: + cmd: | + openssl verify -CAfile /etc/logstash/certs/standalone-ca.crt \ + /etc/logstash/certs/{{ inventory_hostname }}-server.crt + executable: /bin/bash + register: _server_verify + changed_when: false - - name: Assert server key exists + - name: Assert verify returned "OK" ansible.builtin.assert: that: - - server_key.stat.exists - fail_msg: "Server key not found in /etc/logstash/certs/" + - "'OK' in _server_verify.stdout" + fail_msg: "openssl verify did not return OK: {{ _server_verify.stdout }}" + + # --- Server key is PKCS#8 PEM (Logstash requirement) --- - - name: Verify server key is unencrypted PKCS#8 PEM - ansible.builtin.command: head -1 /etc/logstash/certs/{{ ansible_facts.hostname }}.key + - name: Read server key header + ansible.builtin.command: + cmd: head -1 /etc/logstash/certs/{{ inventory_hostname }}.key register: _server_key_header changed_when: false - - name: Assert PKCS#8 header + - name: Assert server key is unencrypted PKCS#8 PEM ansible.builtin.assert: that: - _server_key_header.stdout == '-----BEGIN PRIVATE KEY-----' fail_msg: >- - Expected unencrypted PKCS#8 PEM (-----BEGIN PRIVATE KEY-----), - got {{ _server_key_header.stdout }}. - - - name: Check CA certificate was copied - ansible.builtin.stat: - path: /etc/logstash/certs/ca.crt - register: ca_cert - - - name: Assert CA certificate exists - ansible.builtin.assert: - that: - - ca_cert.stat.exists - fail_msg: "CA certificate not found in /etc/logstash/certs/" + Expected PKCS#8 PEM (-----BEGIN PRIVATE KEY-----), got + {{ _server_key_header.stdout }}. The beats input plugin requires PKCS#8. - # --- Input SSL verification --- + # --- Keystore unlocks with the configured passphrase --- - - name: Read input configuration - ansible.builtin.slurp: - src: /etc/logstash/conf.d/main/10-input.conf - register: input_content + - name: Confirm PKCS12 keystore unlocks with the test passphrase + ansible.builtin.command: + cmd: >- + openssl pkcs12 -in /etc/logstash/certs/keystore.pfx + -nokeys -passin pass:StandaloneTestPass + register: _pfx_check + changed_when: false + failed_when: _pfx_check.rc != 0 + + - name: Confirm PKCS12 keystore REJECTS a wrong passphrase + ansible.builtin.command: + cmd: >- + openssl pkcs12 -in /etc/logstash/certs/keystore.pfx + -nokeys -passin pass:wrong-passphrase + register: _pfx_bad_check + changed_when: false + failed_when: false - - name: Verify beats input with SSL is configured + - name: Assert wrong-passphrase open failed ansible.builtin.assert: that: - - "'beats {' in (input_content.content | b64decode)" - - "'port => 5044' in (input_content.content | b64decode)" - - "'ssl_enabled' in (input_content.content | b64decode) or 'ssl => true' in (input_content.content | b64decode)" - fail_msg: "Beats input with SSL not properly configured" + - _pfx_bad_check.rc != 0 + fail_msg: >- + keystore.pfx accepted a wrong passphrase — the role generated + an unencrypted (or null-password) keystore. - # --- Port check --- + # --- Listening + TLS handshake --- - - name: Check Logstash is listening on port 5044 + - name: Wait for Logstash beats input on 5044 ansible.builtin.wait_for: port: 5044 - timeout: 120 + timeout: 180 state: started - register: port_check + register: _port_check ignore_errors: true - - name: Show diagnostics on port check failure - when: port_check is failed - block: - - name: Show Logstash log after failure - ansible.builtin.shell: >- - journalctl -u logstash --no-pager -n 100 2>/dev/null || - tail -100 /var/log/logstash/logstash-plain.log 2>/dev/null || - echo "No logs found" - register: logstash_log - changed_when: false - - - name: Display failure log # noqa: no-handler - ansible.builtin.debug: - var: logstash_log.stdout_lines - - - name: Fail with context - ansible.builtin.fail: - msg: "Logstash port 5044 did not open within timeout." - - # --- Config syntax validation --- - - - name: Get installed Logstash version - ansible.builtin.shell: >- - set -o pipefail && - (v=$(dpkg-query -W -f='${Version}' logstash 2>/dev/null) && - echo "${v}" | sed 's/^[0-9]*://;s/-.*$//') || - rpm -q --qf '%{VERSION}' logstash 2>/dev/null - args: + - name: TLS handshake against Logstash 5044 (signed by standalone CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5044 \ + -CAfile /etc/logstash/certs/standalone-ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' executable: /bin/bash - register: logstash_version + register: _ls_tls_probe changed_when: false + when: _port_check is not failed - - name: Verify Logstash config syntax - ansible.builtin.command: >- - /usr/share/logstash/bin/logstash - --path.settings=/etc/logstash - --config.test_and_exit - -f /etc/logstash/conf.d/main/ - become: "{{ logstash_version.stdout is version('9.0.0', '>=', version_type='loose') }}" - become_method: ansible.builtin.su - become_user: "{{ 'logstash' if logstash_version.stdout is version('9.0.0', '>=', version_type='loose') else omit }}" - become_flags: '-s /bin/sh' - register: config_test - changed_when: false - failed_when: config_test.rc != 0 + - name: Show diagnostics if port did not open + when: _port_check is failed + block: + - name: Show recent Logstash log + ansible.builtin.shell: + cmd: >- + journalctl -u logstash --no-pager -n 100 2>/dev/null + || tail -100 /var/log/logstash/logstash-plain.log 2>/dev/null + || echo "no logs" + executable: /bin/bash + register: _ls_log + changed_when: false + - name: Fail with log context + ansible.builtin.fail: + msg: "Port 5044 did not open. Recent log:\n{{ _ls_log.stdout }}" diff --git a/molecule/plugins/converge.yml b/molecule/plugins/converge.yml deleted file mode 100644 index 1a9d8e60..00000000 --- a/molecule/plugins/converge.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -# The workaround for arbitrarily named role directory is important because the git repo has one name and the role within it another -# Found at: https://github.com/ansible-community/molecule/issues/1567#issuecomment-436876722 -- name: Converge - hosts: localhost - tasks: - # - # Test modules - # - - name: Test - oddly.elasticstack.cert_info: - path: files/es-ca/elastic-stack-ca.p12 - passphrase: PleaseChangeMe - register: test - - name: Debug - ansible.builtin.debug: - msg: "{{ test }}" - - name: Test required parameters (missing path) # noqa: args[module] - oddly.elasticstack.cert_info: - passphrase: PleaseChangeMe - failed_when: false - - name: Test wrong path - oddly.elasticstack.cert_info: - path: es-ca-wrong - passphrase: PleaseChangeMe - failed_when: false - - name: Debug with to_datetime() - strip timezone for naive datetime arithmetic - ansible.builtin.debug: - msg: >- - "{{ ((test.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')) - (ansible_facts.date_time.date | to_datetime('%Y-%m-%d'))).days }}" - - name: Test wrong passphrase - oddly.elasticstack.cert_info: - path: files/es-ca/elastic-stack-ca.p12 - passphrase: PleaseChangeMe-wrong - failed_when: false - - name: Test no passphrase - oddly.elasticstack.cert_info: - path: files/es-ca/elastic-stack-ca.p12 - failed_when: false - - name: Test no parameters # noqa: args[module] - oddly.elasticstack.cert_info: - failed_when: false diff --git a/molecule/shared/verify_es_health.yml b/molecule/shared/verify_es_health.yml index da02f9b1..3ec65067 100644 --- a/molecule/shared/verify_es_health.yml +++ b/molecule/shared/verify_es_health.yml @@ -2,18 +2,21 @@ # Check Elasticsearch cluster health via the REST API. # Required variables: # elastic_pass: registered result with .stdout containing the password +# (omit auth altogether by setting _verify_es_scheme: http, +# in which case elastic_pass is not consulted) # Optional variables: +# _verify_es_scheme: 'http' or 'https' (default: https) # _verify_es_port: HTTP port (default: 9200) # _verify_es_status: expected status — 'green' or 'yellow' (default: green) # _verify_run_once: run once across hosts (default: false) # _verify_when: additional when condition (default: true) - name: Elasticsearch health check ansible.builtin.uri: - url: "https://localhost:{{ _verify_es_port | default(elasticstack_elasticsearch_http_port | default(9200)) }}/_cluster/health" + url: "{{ _verify_es_scheme | default('https') }}://localhost:{{ _verify_es_port | default(elasticstack_elasticsearch_http_port | default(9200)) }}/_cluster/health" method: GET - force_basic_auth: true - user: elastic - password: "{{ elastic_pass.stdout }}" + force_basic_auth: "{{ (_verify_es_scheme | default('https')) == 'https' }}" + user: "{{ 'elastic' if (_verify_es_scheme | default('https')) == 'https' else omit }}" + password: "{{ elastic_pass.stdout if (_verify_es_scheme | default('https')) == 'https' else omit }}" return_content: true status_code: 200 validate_certs: false diff --git a/scripts/check-ci-coverage.sh b/scripts/check-ci-coverage.sh index 7754177e..efbc225d 100755 --- a/scripts/check-ci-coverage.sh +++ b/scripts/check-ci-coverage.sh @@ -9,8 +9,9 @@ WORKFLOWS_DIR="$REPO_ROOT/.github/workflows" MOLECULE_DIR="$REPO_ROOT/molecule" EXIT_CODE=0 -# Scenarios that are not standalone tests (utility dirs, shared includes) -EXCLUDED_SCENARIOS="default shared plugins" +# Scenarios that are not standalone tests (utility dirs, shared includes, +# or playbooks invoked directly by ansible-playbook rather than `molecule`) +EXCLUDED_SCENARIOS="default shared cert_info_module" echo "=== Molecule scenario CI coverage check ===" echo diff --git a/scripts/wait-for-memory.sh b/scripts/wait-for-memory.sh new file mode 100755 index 00000000..0d6cff90 --- /dev/null +++ b/scripts/wait-for-memory.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Coordinated memory admission gate for the molecule runner pool. +# +# Each scenario reserves its memory_mb on the runner host before converge, +# and releases the reservation in a post-step. A flock-serialised reader +# sums in-flight reservations and only admits a new job when: +# +# MemAvailable - sum(reservations) >= scenario_mb + buffer +# +# Reservations are files under $GATE_DIR keyed by RUNNER_NAME (so each runner +# instance can hold at most one). Stale files older than $TTL_SEC are +# garbage-collected during the lock window — covers crashed jobs that +# never reach the release step. +# +# Usage: +# wait-for-memory.sh acquire [timeout_s] +# wait-for-memory.sh release +# +# Env: +# RUNNER_NAME (auto-set by GitHub Actions; required) +# MOLECULE_GATE_DIR (default /var/lib/molecule-gate, falls back to /tmp/molecule-gate) +# MOLECULE_GATE_TTL (default 3600 — drop reservations older than this) +# WAIT_FOR_MEMORY_BUFFER_MB (default 2048 — extra headroom over scenario_mb) + +set -euo pipefail + +action="${1:?usage: $0 [args...]}" + +GATE_DIR="${MOLECULE_GATE_DIR:-/var/lib/molecule-gate}" +mkdir -p "$GATE_DIR" 2>/dev/null || { + GATE_DIR=/tmp/molecule-gate + mkdir -p "$GATE_DIR" +} +chmod 0777 "$GATE_DIR" 2>/dev/null || true + +LOCK="$GATE_DIR/.lock" +TTL_SEC="${MOLECULE_GATE_TTL:-3600}" +buffer_mb="${WAIT_FOR_MEMORY_BUFFER_MB:-2048}" + +runner="${RUNNER_NAME:-runner-$$}" +my_resv="$GATE_DIR/r.${runner}" + +# --- Required MB per scenario (sum of memory_mb across all platforms) --- +# Update this table when a scenario's memory_mb changes. +declare -A REQ=( + [beats_advanced]=2048 + [beats_default]=4096 + [beats_peculiar]=2048 + [beats_security]=8192 + [cert_renewal]=10752 + [elasticsearch_cert_content]=4096 + [elasticsearch_custom]=4096 + [elasticsearch_custom_certs]=4096 + [elasticsearch_custom_certs_minimal]=4096 + [elasticsearch_default]=8192 + [elasticsearch_diagnostics]=4096 + [elasticsearch_no-security]=8192 + [elasticsearch_roles_calculation]=16384 + [elasticsearch_security_api]=4096 + [elasticsearch_upgrade_8to9]=8192 + [elasticsearch_upgrade_8to9_single]=4096 + [elasticstack_default]=20480 + [es_kibana]=13824 + [kibana_custom]=8192 + [kibana_custom_certs]=8192 + [kibana_default]=4096 + [logstash_advanced]=2048 + [logstash_centralized_pipelines]=2048 + [logstash_custom_pipeline]=2048 + [logstash_default]=2048 + [logstash_elasticsearch]=8192 + [logstash_external_certs]=2048 + [logstash_standalone_certs]=2048 + [logstash_ssl]=2048 + [repos_default]=1024 +) + +# --- Helpers --- + +# Sum reservations under exclusive lock; garbage-collect stale entries. +# Echoes "" to stdout. Caller must hold the lock or call this +# inside the locked section. +sum_reservations() { + local now total=0 mtime mb + now=$(date +%s) + shopt -s nullglob + for f in "$GATE_DIR"/r.*; do + mtime=$(stat -c %Y "$f" 2>/dev/null || echo 0) + if [ $(( now - mtime )) -gt "$TTL_SEC" ]; then + rm -f "$f" + continue + fi + mb=$(awk '{print $1+0; exit}' "$f" 2>/dev/null || echo 0) + total=$(( total + mb )) + done + shopt -u nullglob + echo "$total" +} + +case "$action" in + acquire) + scenario="${2:?usage: $0 acquire [timeout_s]}" + timeout_s="${3:-1800}" + required="${REQ[$scenario]:-4096}" + threshold=$(( required + buffer_mb )) + + printf 'molecule-gate[%s]: acquire scenario=%s required=%dMB buffer=%dMB threshold=%dMB timeout=%ds\n' \ + "$runner" "$scenario" "$required" "$buffer_mb" "$threshold" "$timeout_s" + + deadline=$(( $(date +%s) + timeout_s )) + attempt=0 + while :; do + attempt=$(( attempt + 1 )) + # Critical section: read MemAvailable, sum reservations, optionally claim. + exec 9>"$LOCK" + flock 9 + reserved=$(sum_reservations) + available=$(awk '/^MemAvailable:/{print int($2/1024)}' /proc/meminfo) + effective=$(( available - reserved )) + if [ "$effective" -ge "$threshold" ]; then + printf '%d %s\n' "$required" "$scenario" > "$my_resv" + flock -u 9 + printf 'molecule-gate[%s]: ADMITTED MemAvailable=%dMB reserved=%dMB effective=%dMB required=%dMB (attempt %d)\n' \ + "$runner" "$available" "$reserved" "$effective" "$threshold" "$attempt" + exit 0 + fi + flock -u 9 + + now=$(date +%s) + if [ "$now" -ge "$deadline" ]; then + # Take the slot anyway under the same lock — better one OOM than a stuck queue. + exec 9>"$LOCK" + flock 9 + printf '%d %s\n' "$required" "$scenario" > "$my_resv" + flock -u 9 + printf 'molecule-gate[%s]: TIMEOUT after %ds (effective=%dMB needed=%dMB) — proceeding without headroom\n' \ + "$runner" "$timeout_s" "$effective" "$threshold" >&2 + exit 0 + fi + printf 'molecule-gate[%s]: waiting (available=%dMB reserved=%dMB effective=%dMB needed=%dMB, attempt %d)\n' \ + "$runner" "$available" "$reserved" "$effective" "$threshold" "$attempt" + sleep 30 + done + ;; + + release) + exec 9>"$LOCK" + flock 9 + if [ -f "$my_resv" ]; then + rm -f "$my_resv" + printf 'molecule-gate[%s]: released\n' "$runner" + else + printf 'molecule-gate[%s]: nothing to release\n' "$runner" + fi + flock -u 9 + ;; + + *) + echo "usage: $0 [args...]" >&2 + exit 2 + ;; +esac