update

README.md

@@ -8,7 +8,7 @@ The Ansible Datasaker role installs and configures the Datasaker Agent and integ
 
 - Requires Ansible v2.6+.
 - Supports most Debian Linux distributions.
-- Supports Amazon Linux 2023 distributions.
+- Supports Amazon Linux 2 distributions.
 
 ### Installation
 

defaults/main.yml

@@ -35,8 +35,7 @@ apt_usr_share_keyring: "/usr/share/keyrings/datasaker-archive-keyring.gpg"
 dsk_public_gpg_key: "https://dsk-agent-s3.s3.ap-northeast-2.amazonaws.com/dsk-agent-s3/public/public.gpg.key"
 
 datasaker_agents: ['dsk-node-agent']
-
-datasaker_docker_agents: ['dsk-docker-node-agent']
+datasaker_docker_agents: []
 
 datasaker_docker_config_path: "~/.datasaker"
 datasaker_docker_global_config: "{{ datasaker_docker_config_path }}/config.yml"
@@ -71,4 +70,4 @@ postgres_agent_log_level: "INFO"
 plan_postgres_agent_log_level: "INFO"
 
 uninstall: False
-datasaker_clean: False
+datasaker_clean: False

tasks/check-agent.yml

@@ -9,7 +9,8 @@
 
 - name: "Check dsk-log-agent"
   include_tasks: dsk-log-agent.yml
-  when: '"dsk-log-agent" in datasaker_agents'
+  when: 
+  - '"dsk-log-agent" in datasaker_agents'
 
 - name: "Check dsk-postgres-agent"
   include_tasks: dsk-postgres-agent.yml

tasks/dsk-common.yml

@@ -30,7 +30,7 @@
   when:
   - datasaker_docker_agents
 
-- name: "Touch agent config"
+- name: "Create agent config"
   ansible.builtin.file:
     path: "{{ datasaker_docker_global_config }}"
     state: touch

tasks/dsk-debian-pkg.yml

@@ -4,6 +4,10 @@
     src: global-config.yml.j2
     dest: "/etc/datasaker/global-config.yml"
 
+- set_fact:
+    datasaker_agents: "{{ datasaker_agents | reject('equalto', 'dsk-log-agent') | list }}"
+  when: ansible_facts.distribution_version != "18.04"
+
 - name: "Create temporary directory for key manipulation"
   tempfile:
     state: directory
@@ -67,10 +71,29 @@
 - name: "Install datasaker agent"
   apt:
     name: "{{ item }}"
-    state: present
+    state: latest
     update_cache: yes
   with_items:
     - "{{ datasaker_agents }}"
-  notify:
-    - Reload systemd configuration
-    - Restart {{ item }} service
+  #  - Restart {{ item }} service
+  ignore_errors: true
+
+- name: "Install datasaker log agent"
+  apt:
+    name: "td-agent"
+    state: latest
+    update_cache: yes
+  when: '"dsk-log-agent" in datasaker_agents'
+
+- name: Reload systemd configuration
+  ansible.builtin.systemd:
+    daemon_reload: True
+
+- name: Restart dsk-agent service
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: restarted
+  with_items:
+    - "{{ datasaker_agents }}"
+  ignore_errors: true    

tasks/dsk-docker-log-agent.yml

@@ -9,20 +9,20 @@
   with_items:
     - "{{ datasaker_docker_path }}/agent/dsk-log-agent"
 
-- name: "Touch agent config"
+- name: "create agent config"
   ansible.builtin.file:
-    path: "{{ datasaker_docker_path }}/log-agent-config.yml"
+    path: "{{ datasaker_docker_path }}/dsk-docker-log-agent-config.yml"
     state: touch
 
 - name: "Setting dsk-log-agent config"
   template:
-    src: log-agent-config.yml.j2
-    dest: "{{ datasaker_docker_path }}/log-agent-config.yml"
+    src: docker-log-agent-config.yml.j2
+    dest: "{{ datasaker_docker_path }}/dsk-docker-log-agent-config.yml"
 
 - name: Run Datasaker Log Agent
   docker_container:
     name: dsk-docker-log-agent
-    image: "datasaker/dsk-log-agent:{{ trace_agent_image_tag }}"
+    image: "datasaker/dsk-log-agent:{{ log_agent_image_tag }}"
     state: started
     detach: true
     restart_policy: "{{ log_agent_restart_policy }}"
@@ -32,6 +32,6 @@
     volumes:
       - "{{ datasaker_docker_path }}:/var/datasaker/"
       - "{{ datasaker_docker_global_config }}:/etc/datasaker/global-config.yml:ro"
-      - "{{ datasaker_docker_path }}/log-agent-config.yml:/etc/datasaker/dsk-log-agent/agent-config.yml:ro"
+      - "{{ datasaker_docker_path }}/dsk-docker-log-agent-config.yml:/etc/datasaker/dsk-log-agent/agent-config.yml:ro"
       - "{{ item }}"
   loop: "{{ custom_log_volume | default(['{{ docker_default_path }}:/var/log/sample/:ro']) }}"

tasks/dsk-docker-postgres-agent.yml

@@ -10,20 +10,20 @@
     - "{{ datasaker_docker_path }}/agent/dsk-postgres-agent"
     - "{{ datasaker_docker_path }}/agent/dsk-plan-postgres-agent"
 
-- name: "Touch agent config"
+- name: "Create agent config"
   ansible.builtin.file:
-    path: "{{ datasaker_docker_path }}/dsk-postgres-config.yml"
+    path: "{{ datasaker_docker_path }}/dsk-docker-postgres-config.yml"
     state: touch
 
 - name: "Setting dsk-postgres-agent config"
   template:
     src: docker-postgres-agent-config.yml.j2
-    dest: "{{ datasaker_docker_path }}/dsk-postgres-config.yml"
+    dest: "{{ datasaker_docker_path }}/dsk-docker-postgres-config.yml"
 
 - name: "Setting plan-dsk-postgres-agent config"
   template:
-    src: plan-postgres-agent-config.yml.j2
-    dest: "{{ datasaker_docker_path }}/dsk-plan-postgres-config.yml"
+    src: docker-plan-postgres-agent-config.yml.j2
+    dest: "{{ datasaker_docker_path }}/dsk-docker-plan-postgres-config.yml"
 
 - name: "Run dsk-postgres-agent container"
   docker_container:
@@ -40,7 +40,7 @@
     volumes:
       - "{{ datasaker_docker_path }}:/var/datasaker/"
       - "{{ datasaker_docker_global_config }}:/etc/datasaker/global-config.yml:ro"
-      - "{{ datasaker_docker_path }}/dsk-postgres-config.yml:/etc/datasaker/dsk-postgres-agent/agent-config.yml:ro"
+      - "{{ datasaker_docker_path }}/dsk-docker-postgres-config.yml:/etc/datasaker/dsk-postgres-agent/agent-config.yml:ro"
 
 - name: "Run dsk-plan-postgres-agent container"
   docker_container:
@@ -54,4 +54,4 @@
     volumes:
       - "{{ datasaker_docker_path }}:/var/datasaker/"
       - "{{ datasaker_docker_global_config }}:/etc/datasaker/global-config.yml:ro"
-      - "{{ datasaker_docker_path }}/dsk-plan-postgres-config.yml:/etc/datasaker/dsk-plan-postgres-agent/agent-config.yml:ro"
+      - "{{ datasaker_docker_path }}/dsk-docker-plan-postgres-config.yml:/etc/datasaker/dsk-plan-postgres-agent/agent-config.yml:ro"

tasks/dsk-docker-trace-agent.yml

@@ -9,15 +9,15 @@
   with_items:
     - "{{ datasaker_docker_path }}/agent/dsk-trace-agent"
 
-- name: "Touch agent config"
+- name: "Create agent config"
   ansible.builtin.file:
-    path: "{{ datasaker_docker_path }}/trace-agent-config.yml"
+    path: "{{ datasaker_docker_path }}/dsk-docker-trace-agent-config.yml"
     state: touch
 
 - name: "Setting dsk-trace-agent config"
   template:
     src: trace-agent-config.yml.j2
-    dest: "{{ datasaker_docker_path }}/trace-agent-config.yml"
+    dest: "{{ datasaker_docker_path }}/dsk-docker-trace-agent-config.yml"
 
 - name: "Run Datasaker trace Agent"
   ansible.builtin.docker_container:
@@ -34,4 +34,4 @@
     volumes:
       - "{{ datasaker_docker_path }}:/var/datasaker/"
       - "{{ datasaker_docker_global_config }}:/etc/datasaker/global-config.yml:ro"
-      - "{{ datasaker_docker_path }}/trace-agent-config.yml:/etc/datasaker/dsk-trace-agent/agent-config.yml:ro"
+      - "{{ datasaker_docker_path }}/dsk-docker-trace-agent-config.yml:/etc/datasaker/dsk-trace-agent/agent-config.yml:ro"

tasks/dsk-redhat-pkg.yml

@@ -28,6 +28,13 @@
   with_items:
     - "{{ datasaker_agents }}"
 
+- name: "Install datasaker log agent"
+  apt:
+    name: "td-agent"
+    state: latest
+    update_cache: yes
+  when: '"dsk-log-agent" in datasaker_agents'
+
 - name: "Install datasaker agent"
   yum:
     name: "{{ item }}"
@@ -38,3 +45,4 @@
   notify:
     - Reload systemd configuration
     - Restart {{ item }} service
+  ignore_errors: true

tasks/remove-datasaker.yml

@@ -18,7 +18,7 @@
     - datasaker_agents
 
 - name: Stop agent service
-  service:
+  ansible.builtin.systemd:
     name: "{{ item }}"
     enabled: true
     state: stopped
@@ -35,10 +35,27 @@
     update_cache: yes
   with_items:
     - "{{ datasaker_agents }}"
-  when: 
+  when:
+    - ansible_facts.os_family == "Debian"
     - datasaker_agents
+  notify:
+    - Reload systemd configuration
   ignore_errors: true
 
+- name: "Uninstall datasaker agent"
+  yum:
+    name: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ datasaker_agents }}"
+  when:
+    - ansible_facts.os_family == "RedHat"
+    - datasaker_agents
+  notify:
+    - Reload systemd configuration    
+  ignore_errors: true
+
+
 - name: Stop the Docker container
   docker_container:
     name: "{{ item }}"

templates/docker-log-agent-config.yml.j2

@@ -2,30 +2,31 @@ agent:
   metadata:
     agent_name: "{{ log_agent_name | default('dsk-log-agent') }}"
   collect:
-{% for collect_item in log_collects %}
+{% for collect_item in log_collects | default([{'paths': ['/var/log/sample/*/*.log'], 'exclude_paths': [], 'keywords': [], 'tag': 'Default', 'service': {'name': 'default', 'category': 'etc', 'type': 'etc'}}]) %}
     - paths:
 {% for path in collect_item.paths %}
-        - "{{ path | default('/var/log/sample/*/*.log') }}"
+        - "{{ path }}"
 {% endfor %}
-{% if collect_item.exclude_paths is defined %}
       exclude_paths:
+{% if collect_item.exclude_paths | default([]) %}
 {% for exclude_path in collect_item.exclude_paths %}
-        - "{{ exclude_path | default('[]') }}"
+        - "{{ exclude_path }}"
 {% endfor %}
 {% endif %}
-{% if collect_item.keywords is defined %}
       keywords:
+{% if collect_item.keywords | default([]) %}
 {% for keyword in collect_item.keywords %}
-        - "{{ keyword | default('[]') }}"
+        - "{{- keyword -}}"
 {% endfor %}
 {% endif %}
-      tag: "{{ collect_item.tag | default('Default') }}"
+{% if collect_item.tag is defined %}
+      tag: "{{ collect_item.tag }}"
+{% endif %}
       service:
-        name: "{{ collect_item.service.name | default('default') }}"
-        category: "{{ collect_item.service.category | default('etc') }}"
-        type: "{{ collect_item.service.type | default('etc') }}"
+        name: "{{ collect_item.service.name }}"
+        category: "{{ collect_item.service.category }}"
+        type: "{{ collect_item.service.type }}"
 {% if collect_item.service.address is defined %}
         address: "{{ collect_item.service.address }}"
 {% endif %}
-{% endfor %}
-
+{% endfor %}

templates/docker-plan-postgres-agent-config.yml.j2

@@ -0,0 +1,13 @@
+agent:
+  metadata:
+    agent_name: "{{ plan_postgres_agent_name | default('dsk-plan-postgres-agent') }}"
+  data_source_name:
+    user: "{{ plan_postgres_user_name | default('') }}"
+    password: "{{ plan_postgres_user_password | default('') }}"
+    address: "{{ plan_postgres_database_address | default('') }}"
+    port: "{{ plan_postgres_database_port | default('') }}"
+    DBName: "{{ plan_postgres_database_name | default('') }}"
+  explain:
+    scrape_interval: {{ plan_postgres_scrape_interval | default('30s') }}
+    scrape_timeout: {{ plan_postgres_scrape_timeout | default('5s') }}
+    slow_query_standard: {{ plan_postgres_slow_query_standard | default('5s') }}

templates/log-agent-config.yml.j2

@@ -2,30 +2,31 @@ agent:
   metadata:
     agent_name: "{{ log_agent_name | default('dsk-log-agent') }}"
   collect:
-{% for collect_item in log_collects %}
+{% for collect_item in log_collects | default([{'paths': ['/var/log/*.log'], 'exclude_paths': [], 'keywords': [], 'tag': 'Default', 'service': {'name': 'default', 'category': 'etc', 'type': 'etc'}}]) %}
     - paths:
 {% for path in collect_item.paths %}
-        - "{{ path | default('/var/log/*.log') }}"
+        - "{{ path }}"
 {% endfor %}
-{% if collect_item.exclude_paths is defined %}
       exclude_paths:
+{% if collect_item.exclude_paths | default([]) %}
 {% for exclude_path in collect_item.exclude_paths %}
-        - "{{ exclude_path | default('[]') }}"
+        - "{{ exclude_path }}"
 {% endfor %}
 {% endif %}
-{% if collect_item.keywords is defined %}
       keywords:
+{% if collect_item.keywords | default([]) %}
 {% for keyword in collect_item.keywords %}
-        - "{{ keyword | default('[]') }}"
+        - "{{- keyword -}}"
 {% endfor %}
 {% endif %}
-      tag: "{{ collect_item.tag | default('Default') }}"
+{% if collect_item.tag is defined %}
+      tag: "{{ collect_item.tag }}"
+{% endif %}
       service:
-        name: "{{ collect_item.service.name | default('default') }}"
-        category: "{{ collect_item.service.category | default('etc') }}"
-        type: "{{ collect_item.service.type | default('etc') }}"
+        name: "{{ collect_item.service.name }}"
+        category: "{{ collect_item.service.category }}"
+        type: "{{ collect_item.service.type }}"
 {% if collect_item.service.address is defined %}
         address: "{{ collect_item.service.address }}"
 {% endif %}
-{% endfor %}
-
+{% endfor %}

templates/trace-agent-config.yml.j2

@@ -4,4 +4,4 @@ agent:
     cluster_id: "{{ trace_agent_cluster_id | default('unknown_cluster') }}"
   option:
     collector_config: 
-      sampling_rate: "{{ trace_sampling_rate | default('10') }}"
+      sampling_rate: {{ trace_sampling_rate | default(10) | int }}