Bug fix/slack channels pagination

connection.yml.sample

@@ -44,11 +44,8 @@ sources:
         - table1
         - table2
       exclude_columns:
-        - column1
-        - column2
-  text:
-    profile1:
-      text: "Hello World HHWPK6943Q"
+         - column1
+         - column2
   postgresql:
     postgresql_example:
       host: YOUR_POSTGRESQL_HOST
@@ -84,17 +81,8 @@ sources:
         - private
         - venv
         - node_modules
-
-  slack:
-    slack_example:
-      token: xoxp-XXXXXXXXXXXXXXXXXXXXXXXXX # get your slack app these permissiosn https://api.slack.com/methods/team.info and https://api.slack.com/methods/conversations.list
-      channel_types: "public_channel,private_channel"
-      # Optional: List of channel names to check
-      # channel_names:
-      #   - general
-      #   - random
 
-  gdrive:
+ gdrive:
     drive_example:
       folder_name:
       credentials_file: /Users/kumarohit/Downloads/client_secret.json ## this will be oauth app json file
@@ -114,3 +102,14 @@ sources:
       exclude_patterns:
         - .pdf
         - .docx
+  text:
+    profile1:
+      text: "Hello World HHXXXXX"
+  slack:
+    slack_example:
+      channel_types: "public_channel,private_channel"
+      token: xoxp-XXXXXXXXXXXXXXXXXXXXXXXXX
+      archived_channels: True ## By default False, set to True if you want to scan archived channels also
+      limit_mins: 15 ## By default 60 mins
+      channel_ids:
+      - XXXXXXXX

hawk_scanner/commands/slack.py

@@ -22,7 +22,7 @@ def connect_slack(args, token):
         system.print_error(args, f"Failed to connect to Slack with error: {e.response['error']}")
         return None
 
-def check_slack_messages(args, client, patterns, profile_name, channel_types, channel_ids=None, limit_mins=60):
+def check_slack_messages(args, client, patterns, profile_name, channel_types, channel_ids=None, limit_mins=60, archived_channels=False):
     results = []
     try:
         team_info = client.team_info()
@@ -41,20 +41,53 @@ def check_slack_messages(args, client, patterns, profile_name, channel_types, ch
 
         # Get all channels of specified types
         channels = []
+
         if not channel_ids:
             system.print_info(args, "Getting all channels because no channel_ids provided")
-            channels = client.conversations_list(types=channel_types)["channels"]
+
+            # Pagination logic to fetch all non-archived channels
+            cursor = None
+            while True:
+                try:
+                    if archived_channels:
+                        system.print_debug(args, f"Considering archived channels, you may want to set archived_channels to False")
+                    else:
+                        system.print_debug(args, f"Skipping archived channels, you may want to set archived_channels to True")
+                    response = client.conversations_list(
+                        types=channel_types, 
+                        limit=1000, 
+                        cursor=cursor, 
+                        exclude_archived=not archived_channels
+                    )
+                    channels.extend(response.get("channels", []))
+
+                    # Update the cursor for the next batch
+                    cursor = response.get("response_metadata", {}).get("next_cursor")
+
+                    if not cursor:  # Break the loop if there are no more channels to fetch
+                        break
+                except SlackApiError as e:
+                    system.print_error(args, f"Failed to fetch channels: {e.response['error']}")
+                    break
         else:
             system.print_info(args, "Getting channels by channel_ids")
             for channel_id in channel_ids:
                 try:
                     channel = client.conversations_info(channel=channel_id)["channel"]
-                    channels.append(channel)
+                    ## if archived_channels is set to True, include archived channels
+                    if archived_channels or not channel.get("is_archived"):
+                        system.print_debug(args, f"Considering archived channels, you may want to set archived_channels to False")
+                        channels.append(channel)
+                    else:
+                        system.print_debug(args, f"Skipping archived channel: {channel_id}")
+
                 except SlackApiError as e:
                     system.print_error(args, f"Failed to fetch channel with id {channel_id} with error: {e.response['error']}")
 
+        # Optional: Print or log the total number of channels fetched
+        system.print_info(args, f"Total channels fetched: {len(channels)}")
         system.print_info(args, f"Found {len(channels)} channels of type {channel_types}")
-        system.print_info(args, f"Checking messages in channels: {', '.join([channel['name'] for channel in channels])}")
+        system.print_debug(args, f"Checking messages in channels: {', '.join([channel['name'] for channel in channels])}")
 
         for channel in channels:
             channel_name = channel["name"]
@@ -211,6 +244,7 @@ def execute(args):
                 channel_types = config.get('channel_types', "public_channel,private_channel")
                 channel_ids = config.get('channel_ids', [])
                 limit_mins = config.get('limit_mins', 60)
+                archived_channels = config.get('archived_channels', False)
 
                 if token:
                     system.print_info(args, f"Checking Slack Profile {key}")
@@ -220,7 +254,7 @@ def execute(args):
 
                 client = connect_slack(args, token)
                 if client:
-                    results += check_slack_messages(args, client, patterns, key, channel_types, channel_ids, limit_mins)
+                    results += check_slack_messages(args, client, patterns, key, channel_types, channel_ids, limit_mins, archived_channels)
         else:
             system.print_error(args, "No Slack connection details found in connection.yml")
     else:

readme.md

@@ -3,7 +3,7 @@
 <p align="center">
 <a href="#description">Description</a> • <a href="#installation">Installation</a> • <a href="#features">Features</a> • <a href="#config">Configuration</a> • <a href="#acknowledgements">Acknowledgements</a><br><br>
 
-[![Publish to PyPI.org](https://github.com/rohitcoder/hawk-eye/actions/workflows/pypi.yml/badge.svg)](https://github.com/rohitcoder/hawk-eye/actions/workflows/pypi.yml) 
+
 <img alt="Static Badge" src="https://img.shields.io/badge/Supports-S3-yellow?logo=amazons3">
 <img alt="Static Badge" src="https://img.shields.io/badge/Supports-GCP-red?logo=googlecloud">
 <img alt="Static Badge" src="https://img.shields.io/badge/Supports-MysQL-green?logo=mysql">
@@ -12,19 +12,29 @@
 <img alt="Static Badge" src="https://img.shields.io/badge/Supports-On Prem-black?logo=amazonec2">
 </p>
 
+
+Join our Slack community for support, discussions, or to contribute!
+
+<a href="https://join.slack.com/t/hawkeyecommunity/shared_invite/zt-2xz0qbo8n-KQQ9UQ1KW2QfaMVDmCWYrw" target="_blank">
+    <img src="https://i.imgur.com/BUtBFwE.png" alt="Join Slack Community" width="150" />
+</a>
+
 <div id="description">
 
-### 🦅 HAWK Eye - Uncover Secrets and PII Across All Platforms in Minutes!
+### 🦅 Hawk Eye - Uncover Secrets and PII Across All Platforms in Minutes!
 
-HAWK Eye is a robust, command-line tool built to safeguard against data breaches and cyber threats. Much like the sharp vision of a hawk, it quickly scans multiple data sources—S3, MySQL, PostgreSQL, MongoDB, CouchDB, Google Drive, Slack, Redis, Firebase, file systems, and Google Cloud buckets (GCS)—for Personally Identifiable Information (PII) and secrets. Using advanced text analysis and OCR techniques, HAWK Eye delves into various document formats like docx, xlsx, pptx, pdf, images (jpg, png, gif), compressed files (zip, tar, rar), and even video files to ensure comprehensive protection across platforms.
+Hawk Eye is a robust, command-line tool built to safeguard against data breaches and cyber threats. Much like the sharp vision of a hawk, it quickly scans multiple data sources—S3, MySQL, PostgreSQL, MongoDB, CouchDB, Google Drive, Slack, Redis, Firebase, file systems, and Google Cloud buckets (GCS)—for Personally Identifiable Information (PII) and secrets. Using advanced text analysis and OCR techniques, HAWK Eye delves into various document formats like docx, xlsx, pptx, pdf, images (jpg, png, gif), compressed files (zip, tar, rar), and even video files to ensure comprehensive protection across platforms.
 
 
 ### Why "HAWK Eye"?
 Like the keen vision of a hawk, this tool enables you to monitor and safeguard your data with precision and accuracy, ensuring data privacy and security.
 </div>
 
 ## Commercial Support
-For commercial support and help with HAWK Eye, please contact us at [LinkedIn](https://linkedin.com/in/rohitcoder) or [Twitter](https://twitter.com/rohitcoder).
+
+For commercial support and help with HAWK Eye, please contact us on [LinkedIn](https://linkedin.com/in/rohitcoder) or [Twitter](https://twitter.com/rohitcoder).
+
+Alternatively, you can reach out to us in our Slack community.
 
 ## HAWK Eye in Action
 
@@ -373,7 +383,7 @@ sources:
     slack_example:
       channel_types: "public_channel,private_channel"
       token: xoxp-XXXXXXXXXXXXXXXXXXXXXXXXX
-      channel_types: "public_channel,private_channel"
+      archived_channels: True ## By default False, set to True if you want to scan archived channels also
       limit_mins: 15 ## By default 60 mins
       channel_ids:
       - XXXXXXXX

setup.py

@@ -1,4 +1,4 @@
-VERSION = "0.3.25"
+VERSION = "0.3.26"
 
 from setuptools import setup, find_packages