TensorFlow: Core Concepts & Tensors

TensorFlow is Google's open-source machine learning framework. TF 2.x uses eager execution by default (compute immediately) and Keras as its primary high-level API. Used for neural networks, computer vision, NLP, and time-series.

TensorFlow vs PyTorch

• TensorFlow: better for production deployment (TFLite, TF Serving, TF.js), stronger mobile support, TensorBoard built-in

• PyTorch: more pythonic, preferred in research, easier debugging, dynamic computation graph

• Both have converged significantly — Keras 3 supports both backends

• JAX: growing alternative (Google), functional style, excellent for custom gradients

Tensors

import tensorflow as tf
import numpy as np

# Create tensors
scalar = tf.constant(3.14)                     # rank-0 tensor
vector = tf.constant([1.0, 2.0, 3.0])         # rank-1, shape (3,)
matrix = tf.constant([[1, 2], [3, 4]])         # rank-2, shape (2, 2)
tensor3d = tf.zeros([batch, height, width])    # shape (b, h, w)

# Tensor properties
matrix.shape     # TensorShape([2, 2])
matrix.dtype     # tf.int32
matrix.numpy()   # convert to numpy array

# Operations (eager by default in TF2)
a = tf.constant([[1.0, 2.0], [3.0, 4.0]])
b = tf.constant([[5.0, 6.0], [7.0, 8.0]])

tf.add(a, b)          # element-wise addition
tf.matmul(a, b)       # matrix multiplication
tf.reduce_sum(a)      # sum all elements
tf.reduce_mean(a, axis=0)  # mean along axis
tf.transpose(a)
tf.reshape(a, [4])    # flatten to [4]

# Variables (trainable parameters)
w = tf.Variable([[0.1, 0.2], [0.3, 0.4]], trainable=True)
w.assign(new_value)
w.assign_add(delta)

Automatic Differentiation

# GradientTape — compute gradients
x = tf.Variable(3.0)

with tf.GradientTape() as tape:
    y = x ** 2 + 2 * x + 1   # y = x² + 2x + 1

dy_dx = tape.gradient(y, x)  # dy/dx = 2x + 2 = 8.0

# Multiple variables
w = tf.Variable(2.0)
b = tf.Variable(1.0)

with tf.GradientTape() as tape:
    y_pred = w * x + b
    loss = (y_pred - 5.0) ** 2

grads = tape.gradient(loss, [w, b])  # [dL/dw, dL/db]

Data Pipelines with tf.data

import tensorflow as tf

# From numpy/lists
dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))

# Efficient pipeline
dataset = (
    dataset
    .shuffle(buffer_size=1000, seed=42)
    .batch(32)
    .prefetch(tf.data.AUTOTUNE)   # overlap preprocessing with training
)

# From files
image_dataset = tf.keras.utils.image_dataset_from_directory(
    'data/images/',
    image_size=(224, 224),
    batch_size=32,
    validation_split=0.2,
    subset='training',
    seed=42,
)

# Map preprocessing
def preprocess(image, label):
    image = tf.cast(image, tf.float32) / 255.0
    return image, label

dataset = dataset.map(preprocess, num_parallel_calls=tf.data.AUTOTUNE)

TensorFlow: Building Models with Keras

Sequential API

import tensorflow as tf
from tensorflow import keras

# Sequential — for simple linear stacks of layers
model = keras.Sequential([
    keras.layers.Input(shape=(784,)),
    keras.layers.Dense(256, activation='relu'),
    keras.layers.Dropout(0.3),
    keras.layers.Dense(128, activation='relu'),
    keras.layers.Dropout(0.3),
    keras.layers.Dense(10, activation='softmax'),  # 10-class output
])

model.summary()  # print layer shapes and param counts

Functional API

# Functional — for multi-input/output, residual connections, shared layers
inputs = keras.Input(shape=(224, 224, 3), name='image')

x = keras.layers.Conv2D(32, 3, activation='relu', padding='same')(inputs)
x = keras.layers.MaxPooling2D()(x)
x = keras.layers.Conv2D(64, 3, activation='relu', padding='same')(x)
x = keras.layers.GlobalAveragePooling2D()(x)
x = keras.layers.Dense(256, activation='relu')(x)
x = keras.layers.Dropout(0.5)(x)
outputs = keras.layers.Dense(num_classes, activation='softmax')(x)

model = keras.Model(inputs, outputs, name='cnn')

Compile & Train

# Compile: optimizer, loss, metrics
model.compile(
    optimizer=keras.optimizers.Adam(learning_rate=1e-3),
    loss='sparse_categorical_crossentropy',  # int labels; use 'categorical_crossentropy' for one-hot
    metrics=['accuracy'],
)

# Train
history = model.fit(
    train_dataset,
    epochs=50,
    validation_data=val_dataset,
    callbacks=[
        keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True),
        keras.callbacks.ModelCheckpoint('best_model.keras', save_best_only=True),
        keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=3),
        keras.callbacks.TensorBoard(log_dir='logs/'),
    ]
)

# Evaluate and predict
loss, acc = model.evaluate(test_dataset)
predictions = model.predict(X_test)     # shape: (n_samples, n_classes)
class_ids = predictions.argmax(axis=1)

Common Layers

• Dense: fully connected layer — most common for structured data

• Conv2D: 2D convolution — images, spatial features

• LSTM / GRU: recurrent layers — sequences, time-series, NLP

• Embedding: maps integer IDs to dense vectors — word embeddings

• BatchNormalization: normalize activations — faster training, better generalization

• Dropout: randomly zero activations during training — regularization

• GlobalAveragePooling2D: reduce spatial dims to single vector — replaces Flatten in CNNs

• MultiHeadAttention: Transformer attention — NLP, vision transformers

Transfer Learning

# Load pretrained model (feature extractor)
base_model = keras.applications.EfficientNetV2B0(
    include_top=False,
    weights='imagenet',
    input_shape=(224, 224, 3),
)
base_model.trainable = False   # freeze — only train new head

# Add custom head
x = base_model.output
x = keras.layers.GlobalAveragePooling2D()(x)
x = keras.layers.Dense(256, activation='relu')(x)
outputs = keras.layers.Dense(num_classes, activation='softmax')(x)

model = keras.Model(base_model.input, outputs)

# Phase 2: unfreeze top layers and fine-tune
base_model.trainable = True
for layer in base_model.layers[:-20]:
    layer.trainable = False

model.compile(optimizer=keras.optimizers.Adam(1e-5), loss='...', metrics=[...])

TensorFlow: Training Optimization

Optimizers

• Adam: adaptive learning rate per parameter — best default choice for most problems

• SGD + momentum: often better final accuracy than Adam with proper lr schedule

• RMSprop: good for RNNs

• AdamW: Adam with weight decay — better regularization than L2 on Adam

• Learning rate: most impactful hyperparameter. Start with 1e-3, tune with lr finder.

# Learning rate schedules
lr_schedule = keras.optimizers.schedules.CosineDecay(
    initial_learning_rate=1e-3,
    decay_steps=total_steps,
    alpha=1e-5  # minimum lr
)

lr_schedule = keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-3,
    decay_steps=1000,
    decay_rate=0.96,
)

model.compile(optimizer=keras.optimizers.Adam(lr_schedule), ...)

Regularization

# L2 regularization
keras.layers.Dense(128, activation='relu',
    kernel_regularizer=keras.regularizers.L2(1e-4))

# Dropout (most common)
keras.layers.Dropout(0.5)   # drop 50% of activations during training

# Batch normalization (before activation)
keras.layers.BatchNormalization()
keras.layers.Dense(128)
keras.layers.Activation('relu')

# Data augmentation (image)
augmentation = keras.Sequential([
    keras.layers.RandomFlip('horizontal'),
    keras.layers.RandomRotation(0.1),
    keras.layers.RandomZoom(0.1),
    keras.layers.RandomContrast(0.1),
])

# Apply during training only
x = augmentation(inputs, training=True)

Custom Training Loop

# For full control over training (research, custom losses)
optimizer = keras.optimizers.Adam(1e-3)
loss_fn = keras.losses.SparseCategoricalCrossentropy()
train_acc_metric = keras.metrics.SparseCategoricalAccuracy()

@tf.function   # compile to graph for speed
def train_step(x_batch, y_batch):
    with tf.GradientTape() as tape:
        logits = model(x_batch, training=True)
        loss = loss_fn(y_batch, logits)

    gradients = tape.gradient(loss, model.trainable_weights)
    optimizer.apply_gradients(zip(gradients, model.trainable_weights))
    train_acc_metric.update_state(y_batch, logits)
    return loss

for epoch in range(epochs):
    for step, (x_batch, y_batch) in enumerate(train_dataset):
        loss = train_step(x_batch, y_batch)

    acc = train_acc_metric.result()
    print(f"Epoch {epoch}: loss={loss:.4f}, acc={acc:.4f}")
    train_acc_metric.reset_state()

GPU & Mixed Precision

# Check GPUs
print(tf.config.list_physical_devices('GPU'))

# Mixed precision (FP16 compute, FP32 weights) — 2-3x faster on modern GPUs
keras.mixed_precision.set_global_policy('mixed_float16')
# Use float32 for output layer:
outputs = keras.layers.Dense(10, dtype='float32')(x)

# Multi-GPU strategy
strategy = tf.distribute.MirroredStrategy()  # single machine, multiple GPUs
with strategy.scope():
    model = build_model()
    model.compile(...)

TensorFlow: Deployment

Saving & Loading Models

# Save in Keras format (recommended for TF/Keras)
model.save('model.keras')
model = keras.models.load_model('model.keras')

# SavedModel format (for TF Serving, TFLite conversion)
model.export('saved_model/')
# or: tf.saved_model.save(model, 'saved_model/')

model = tf.saved_model.load('saved_model/')

# Save/load weights only (fine-tuning, checkpointing)
model.save_weights('weights.h5')
model.load_weights('weights.h5')

# HDF5 format (legacy)
model.save('model.h5')
model = keras.models.load_model('model.h5')

TF Serving (Production API)

# Serve a SavedModel via REST/gRPC
docker run -p 8501:8501 \
  --mount type=bind,source=/path/to/saved_model,target=/models/my_model \
  -e MODEL_NAME=my_model \
  tensorflow/serving

# REST prediction request
curl -X POST http://localhost:8501/v1/models/my_model:predict \
  -H "Content-Type: application/json" \
  -d '{"instances": [[1.0, 2.0, 3.0, ...]]}'

TFLite — Mobile & Edge

# Convert to TFLite
converter = tf.lite.TFLiteConverter.from_saved_model('saved_model/')
# Optional: quantize for smaller/faster model
converter.optimizations = [tf.lite.Optimize.DEFAULT]  # dynamic range quant
# Full integer quantization (fastest on CPU/microcontrollers):
converter.representative_dataset = representative_dataset_gen
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]

tflite_model = converter.convert()

with open('model.tflite', 'wb') as f:
    f.write(tflite_model)

# Run inference with TFLite interpreter
interpreter = tf.lite.Interpreter(model_path='model.tflite')
interpreter.allocate_tensors()

input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
output = interpreter.get_tensor(output_details[0]['index'])

TensorFlow.js

// Run TF models in the browser or Node.js
import * as tf from '@tensorflow/tfjs'
import '@tensorflow/tfjs-backend-webgl'   // GPU in browser

// Load a converted model
const model = await tf.loadLayersModel('/model/model.json')

// Predict
const input = tf.tensor2d([[1, 2, 3, 4]])
const prediction = model.predict(input)
const result = prediction.dataSync()

// Convert from Python SavedModel:
# pip install tensorflowjs
# tensorflowjs_converter --input_format=keras model.keras ./web_model

TensorBoard

# Log metrics, histograms, images during training
tensorboard_cb = keras.callbacks.TensorBoard(
    log_dir='logs/',
    histogram_freq=1,   # log weight histograms every epoch
    write_images=True,  # log model weights as images
)

model.fit(..., callbacks=[tensorboard_cb])

# Launch TensorBoard
# tensorboard --logdir logs/
# Open: http://localhost:6006

• Scalars tab: loss and metrics over epochs

• Histograms: weight distributions — detect vanishing/exploding gradients

• Images: visualize input images or feature maps

• Projector: visualize high-dimensional embeddings with t-SNE/PCA

• Profile: per-step timing — identify bottlenecks in data pipeline or GPU utilization