Hadoop & HDFS

# Core HDFS Configuration (hdfs-site.xml)
<configuration>
  <!-- Block size configuration -->
  <property>
    <name>dfs.blocksize</name>
    <value>268435456</value> <!-- 256MB blocks -->
    <description>Default block size for HDFS files</description>
  </property>
  
  <!-- Replication factor -->
  <property>
    <name>dfs.replication</name>
    <value>3</value>
    <description>Default replication factor</description>
  </property>
  
  <!-- NameNode directories -->
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>file:///hadoop/hdfs/namenode</value>
    <description>NameNode metadata storage</description>
  </property>
  
  <!-- DataNode directories -->
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>file:///hadoop/hdfs/datanode</value>
    <description>DataNode block storage</description>
  </property>
  
  <!-- Security and permissions -->
  <property>
    <name>dfs.permissions.enabled</name>
    <value>true</value>
  </property>
  
  <!-- Heartbeat interval -->
  <property>
    <name>dfs.heartbeat.interval</name>
    <value>3</value>
    <description>DataNode heartbeat interval in seconds</description>
  </property>
</configuration>

# Basic HDFS Operations
# Format NameNode (initial setup only)
hdfs namenode -format

# Start HDFS services
start-dfs.sh

# Basic file operations
hdfs dfs -mkdir /user/data
hdfs dfs -put local_file.txt /user/data/
hdfs dfs -ls /user/data/
hdfs dfs -cat /user/data/local_file.txt
hdfs dfs -get /user/data/local_file.txt downloaded_file.txt

# Cluster health check
hdfs fsck / -files -blocks -locations
hdfs dfsadmin -report

# Python HDFS Client Example
from hdfs import InsecureClient
import os

class HDFSManager:
    def __init__(self, namenode_url='http://localhost:9870'):
        self.client = InsecureClient(namenode_url)
    
    def upload_file(self, local_path, hdfs_path, overwrite=False):
        """Upload local file to HDFS"""
        try:
            with open(local_path, 'rb') as local_file:
                self.client.write(hdfs_path, local_file, overwrite=overwrite)
            print(f"Uploaded {local_path} to {hdfs_path}")
        except Exception as e:
            print(f"Upload failed: {e}")
    
    def download_file(self, hdfs_path, local_path):
        """Download file from HDFS to local filesystem"""
        try:
            with self.client.read(hdfs_path) as hdfs_file:
                with open(local_path, 'wb') as local_file:
                    for chunk in hdfs_file:
                        local_file.write(chunk)
            print(f"Downloaded {hdfs_path} to {local_path}")
        except Exception as e:
            print(f"Download failed: {e}")
    
    def list_directory(self, hdfs_path='/'):
        """List directory contents"""
        try:
            contents = self.client.list(hdfs_path, status=True)
            for item, status in contents:
                file_type = 'DIR' if status['type'] == 'DIRECTORY' else 'FILE'
                size = status.get('length', 0)
                print(f"{file_type}	{size}	{item}")
        except Exception as e:
            print(f"List failed: {e}")

# Usage example
hdfs = HDFSManager()
hdfs.upload_file('large_dataset.csv', '/user/data/large_dataset.csv')
hdfs.list_directory('/user/data/')