removed unnecessary files causing ci to break
This commit is contained in:
Ramon Perez
parent
bf86512916
commit
1444104418
@ -1,140 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import { dirname } from 'path'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
const blogDir = path.join(__dirname, '..', 'src', 'content', 'blog')
|
||||
|
||||
// Function to convert filename to a valid JavaScript variable name
|
||||
function toVariableName(filename) {
|
||||
// Remove extension and special characters, convert to camelCase
|
||||
const base = path.basename(filename, path.extname(filename))
|
||||
let varName = base
|
||||
.replace(/[-_\s]+(.)?/g, (_, c) => (c ? c.toUpperCase() : ''))
|
||||
.replace(/[^a-zA-Z0-9]/g, '')
|
||||
.replace(/^./, (c) => c.toLowerCase())
|
||||
|
||||
// If the variable name starts with a number, prefix with 'img'
|
||||
if (/^[0-9]/.test(varName)) {
|
||||
varName = 'img' + varName.charAt(0).toUpperCase() + varName.slice(1)
|
||||
}
|
||||
|
||||
return varName
|
||||
}
|
||||
|
||||
// Function to process a single MDX file
|
||||
function processMDXFile(filePath) {
|
||||
console.log(`Processing: ${filePath}`)
|
||||
|
||||
let content = fs.readFileSync(filePath, 'utf-8')
|
||||
|
||||
// Find all image references
|
||||
const imageRegex = /!\[([^\]]*)\]\((\.\/_assets\/[^)]+)\)/g
|
||||
const images = []
|
||||
let match
|
||||
|
||||
while ((match = imageRegex.exec(content)) !== null) {
|
||||
const altText = match[1]
|
||||
const imagePath = match[2]
|
||||
const filename = path.basename(imagePath)
|
||||
const varName = toVariableName(filename) + 'Img'
|
||||
|
||||
// Check if we already have this image
|
||||
if (!images.find((img) => img.varName === varName)) {
|
||||
images.push({
|
||||
varName,
|
||||
path: imagePath,
|
||||
altText,
|
||||
originalMatch: match[0],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (images.length === 0) {
|
||||
console.log(` No images found in ${path.basename(filePath)}`)
|
||||
return
|
||||
}
|
||||
|
||||
console.log(` Found ${images.length} images`)
|
||||
|
||||
// Find where to insert imports (after existing imports or frontmatter)
|
||||
const frontmatterEnd = content.indexOf('---', content.indexOf('---') + 3) + 3
|
||||
let importInsertPosition = frontmatterEnd
|
||||
|
||||
// Check if there are already imports
|
||||
const existingImportRegex = /^import\s+.*$/gm
|
||||
const imports = content.match(existingImportRegex)
|
||||
|
||||
if (imports && imports.length > 0) {
|
||||
// Find the last import
|
||||
const lastImport = imports[imports.length - 1]
|
||||
importInsertPosition = content.indexOf(lastImport) + lastImport.length
|
||||
}
|
||||
|
||||
// Generate import statements
|
||||
const importStatements = images
|
||||
.map((img) => `import ${img.varName} from '${img.path}';`)
|
||||
.join('\n')
|
||||
|
||||
// Insert imports
|
||||
if (imports && imports.length > 0) {
|
||||
// Add to existing imports
|
||||
content =
|
||||
content.slice(0, importInsertPosition) +
|
||||
'\n' +
|
||||
importStatements +
|
||||
content.slice(importInsertPosition)
|
||||
} else {
|
||||
// Add new import section after frontmatter
|
||||
content =
|
||||
content.slice(0, frontmatterEnd) +
|
||||
'\n\n' +
|
||||
importStatements +
|
||||
'\n' +
|
||||
content.slice(frontmatterEnd)
|
||||
}
|
||||
|
||||
// Replace all image references with JSX img tags
|
||||
images.forEach((img) => {
|
||||
// Create regex for this specific image
|
||||
const specificImageRegex = new RegExp(
|
||||
`!\\[([^\\]]*)\\]\\(${img.path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\)`,
|
||||
'g'
|
||||
)
|
||||
|
||||
content = content.replace(specificImageRegex, (match, altText) => {
|
||||
return `<img src={${img.varName}.src} alt="${altText || img.altText}" />`
|
||||
})
|
||||
})
|
||||
|
||||
// Write the updated content back
|
||||
fs.writeFileSync(filePath, content)
|
||||
console.log(` ✓ Updated ${path.basename(filePath)}`)
|
||||
}
|
||||
|
||||
// Process all MDX files in the blog directory
|
||||
function processAllBlogPosts() {
|
||||
const files = fs.readdirSync(blogDir)
|
||||
const mdxFiles = files.filter((file) => file.endsWith('.mdx'))
|
||||
|
||||
console.log(`Found ${mdxFiles.length} MDX files in blog directory\n`)
|
||||
|
||||
mdxFiles.forEach((file) => {
|
||||
const filePath = path.join(blogDir, file)
|
||||
try {
|
||||
processMDXFile(filePath)
|
||||
} catch (error) {
|
||||
console.error(`Error processing ${file}:`, error.message)
|
||||
}
|
||||
})
|
||||
|
||||
console.log('\n✨ All blog posts processed!')
|
||||
}
|
||||
|
||||
// Run the script
|
||||
processAllBlogPosts()
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 885 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 3.9 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 742 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 163 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.0 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 415 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 725 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 130 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 795 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.6 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 162 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 218 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 90 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 37 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 141 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 240 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 586 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 55 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 899 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 117 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 453 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 231 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 143 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 138 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 120 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 673 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 374 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 192 KiB |
@ -1,230 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
src: string;
|
||||
alt: string;
|
||||
caption?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
loading?: 'lazy' | 'eager';
|
||||
class?: string;
|
||||
}
|
||||
|
||||
const {
|
||||
src,
|
||||
alt,
|
||||
caption,
|
||||
width,
|
||||
height,
|
||||
loading = 'lazy',
|
||||
class: className = ''
|
||||
} = Astro.props;
|
||||
|
||||
// Handle different image path formats
|
||||
let imageSrc = src;
|
||||
|
||||
// If the path starts with ./ or ../, it's a relative path from the MDX file
|
||||
if (src.startsWith('./') || src.startsWith('../')) {
|
||||
// Remove the leading ./ or ../
|
||||
imageSrc = src.replace(/^\.\.?\//, '');
|
||||
|
||||
// Prepend the blog content path if it doesn't include it
|
||||
if (!imageSrc.includes('/content/blog/')) {
|
||||
imageSrc = `/src/content/blog/${imageSrc}`;
|
||||
}
|
||||
} else if (!src.startsWith('http') && !src.startsWith('/')) {
|
||||
// For paths without ./ prefix, assume they're relative to blog content
|
||||
imageSrc = `/src/content/blog/${src}`;
|
||||
}
|
||||
---
|
||||
|
||||
<figure class={`blog-image-container ${className}`}>
|
||||
<img
|
||||
src={imageSrc}
|
||||
alt={alt}
|
||||
width={width}
|
||||
height={height}
|
||||
loading={loading}
|
||||
class="blog-image"
|
||||
/>
|
||||
{caption && (
|
||||
<figcaption class="blog-image-caption">{caption}</figcaption>
|
||||
)}
|
||||
</figure>
|
||||
|
||||
<style>
|
||||
.blog-image-container {
|
||||
margin: 2rem 0;
|
||||
text-align: center;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.blog-image {
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
display: block;
|
||||
margin: 0 auto;
|
||||
border-radius: 0.75rem;
|
||||
box-shadow:
|
||||
0 4px 6px -1px rgba(0, 0, 0, 0.1),
|
||||
0 2px 4px -1px rgba(0, 0, 0, 0.06);
|
||||
border: 1px solid var(--sl-color-gray-5);
|
||||
background: var(--sl-color-bg);
|
||||
transition: transform 0.2s ease, box-shadow 0.2s ease;
|
||||
}
|
||||
|
||||
.blog-image:hover {
|
||||
transform: scale(1.01);
|
||||
box-shadow:
|
||||
0 10px 15px -3px rgba(0, 0, 0, 0.1),
|
||||
0 4px 6px -2px rgba(0, 0, 0, 0.05);
|
||||
}
|
||||
|
||||
.blog-image-caption {
|
||||
margin-top: 0.75rem;
|
||||
font-size: 0.875rem;
|
||||
color: var(--sl-color-text-muted);
|
||||
font-style: italic;
|
||||
line-height: 1.5;
|
||||
padding: 0 1rem;
|
||||
}
|
||||
|
||||
/* Loading state */
|
||||
.blog-image[loading="lazy"] {
|
||||
background: linear-gradient(
|
||||
90deg,
|
||||
var(--sl-color-gray-5) 0%,
|
||||
var(--sl-color-gray-4) 50%,
|
||||
var(--sl-color-gray-5) 100%
|
||||
);
|
||||
background-size: 200% 100%;
|
||||
animation: shimmer 1.5s infinite;
|
||||
}
|
||||
|
||||
@keyframes shimmer {
|
||||
0% {
|
||||
background-position: 200% 0;
|
||||
}
|
||||
100% {
|
||||
background-position: -200% 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Dark mode adjustments */
|
||||
:global(.dark) .blog-image {
|
||||
box-shadow:
|
||||
0 4px 6px -1px rgba(0, 0, 0, 0.3),
|
||||
0 2px 4px -1px rgba(0, 0, 0, 0.2);
|
||||
border-color: var(--sl-color-gray-6);
|
||||
}
|
||||
|
||||
:global(.dark) .blog-image:hover {
|
||||
box-shadow:
|
||||
0 10px 15px -3px rgba(0, 0, 0, 0.4),
|
||||
0 4px 6px -2px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
/* Responsive adjustments */
|
||||
@media (max-width: 768px) {
|
||||
.blog-image-container {
|
||||
margin: 1.5rem 0;
|
||||
}
|
||||
|
||||
.blog-image {
|
||||
border-radius: 0.5rem;
|
||||
}
|
||||
|
||||
.blog-image-caption {
|
||||
font-size: 0.8125rem;
|
||||
padding: 0 0.5rem;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 480px) {
|
||||
.blog-image-container {
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
.blog-image {
|
||||
border-radius: 0.375rem;
|
||||
}
|
||||
}
|
||||
|
||||
/* Full-width variant */
|
||||
.blog-image-container.full-width {
|
||||
margin-left: calc(-50vw + 50%);
|
||||
margin-right: calc(-50vw + 50%);
|
||||
max-width: 100vw;
|
||||
}
|
||||
|
||||
.blog-image-container.full-width .blog-image {
|
||||
border-radius: 0;
|
||||
max-width: min(100%, 1400px);
|
||||
}
|
||||
|
||||
/* Inline variant for small icons/images */
|
||||
.blog-image-container.inline {
|
||||
display: inline-block;
|
||||
margin: 0 0.25rem;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
.blog-image-container.inline .blog-image {
|
||||
display: inline-block;
|
||||
max-height: 1.5em;
|
||||
width: auto;
|
||||
border: none;
|
||||
box-shadow: none;
|
||||
border-radius: 0;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.blog-image-container.inline .blog-image:hover {
|
||||
transform: none;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
/* Gallery variant */
|
||||
.blog-image-container.gallery {
|
||||
display: inline-block;
|
||||
width: calc(50% - 0.5rem);
|
||||
margin: 0.5rem 0.25rem;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.blog-image-container.gallery {
|
||||
width: 100%;
|
||||
margin: 1rem 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Error state */
|
||||
.blog-image.error {
|
||||
background: var(--sl-color-gray-5);
|
||||
min-height: 200px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.blog-image.error::after {
|
||||
content: '🖼️ Image not found';
|
||||
position: absolute;
|
||||
color: var(--sl-color-text-muted);
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
// Handle image loading errors
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const images = document.querySelectorAll('.blog-image');
|
||||
|
||||
images.forEach(img => {
|
||||
img.addEventListener('error', function() {
|
||||
this.classList.add('error');
|
||||
this.alt = 'Image could not be loaded';
|
||||
});
|
||||
});
|
||||
});
|
||||
</script>
|
||||
@ -1,87 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
title?: string;
|
||||
description?: string;
|
||||
buttonText?: string;
|
||||
buttonLink?: string;
|
||||
variant?: 'primary' | 'secondary' | 'gradient';
|
||||
align?: 'left' | 'center' | 'right';
|
||||
}
|
||||
|
||||
const {
|
||||
title = "Ready to get started?",
|
||||
description = "Download Jan and start running AI models locally on your device.",
|
||||
buttonText = "Download Jan",
|
||||
buttonLink = "https://jan.ai",
|
||||
variant = 'primary',
|
||||
align = 'center'
|
||||
} = Astro.props;
|
||||
|
||||
const variantClasses = {
|
||||
primary: 'bg-blue-50 dark:bg-blue-900/20 border-blue-200 dark:border-blue-800',
|
||||
secondary: 'bg-gray-50 dark:bg-gray-900/20 border-gray-200 dark:border-gray-800',
|
||||
gradient: 'bg-gradient-to-r from-blue-50 to-purple-50 dark:from-blue-900/20 dark:to-purple-900/20 border-purple-200 dark:border-purple-800'
|
||||
};
|
||||
|
||||
const alignClasses = {
|
||||
left: 'text-left',
|
||||
center: 'text-center',
|
||||
right: 'text-right'
|
||||
};
|
||||
|
||||
const buttonVariantClasses = {
|
||||
primary: 'bg-blue-600 hover:bg-blue-700 text-white',
|
||||
secondary: 'bg-gray-800 hover:bg-gray-900 dark:bg-gray-200 dark:hover:bg-gray-300 text-white dark:text-gray-900',
|
||||
gradient: 'bg-gradient-to-r from-blue-600 to-purple-600 hover:from-blue-700 hover:to-purple-700 text-white'
|
||||
};
|
||||
---
|
||||
|
||||
<div class={`cta-blog ${variantClasses[variant]} ${alignClasses[align]} border rounded-xl p-6 lg:p-8 my-8`}>
|
||||
<div class="max-w-2xl mx-auto">
|
||||
{title && (
|
||||
<h3 class="text-2xl lg:text-3xl font-bold mb-3 text-gray-900 dark:text-gray-100">
|
||||
{title}
|
||||
</h3>
|
||||
)}
|
||||
|
||||
{description && (
|
||||
<p class="text-gray-700 dark:text-gray-300 mb-6 text-lg">
|
||||
{description}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<a
|
||||
href={buttonLink}
|
||||
class={`inline-flex items-center gap-2 px-6 py-3 rounded-lg font-medium transition-all duration-200 ${buttonVariantClasses[variant]} shadow-sm hover:shadow-md`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
{buttonText}
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.cta-blog {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.cta-blog::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: -50%;
|
||||
right: -50%;
|
||||
width: 200%;
|
||||
height: 200%;
|
||||
background: radial-gradient(circle, rgba(255,255,255,0.1) 0%, transparent 70%);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
:global(.dark) .cta-blog::before {
|
||||
background: radial-gradient(circle, rgba(255,255,255,0.05) 0%, transparent 70%);
|
||||
}
|
||||
</style>
|
||||
@ -1,85 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
type?: 'info' | 'warning' | 'error' | 'success' | 'note';
|
||||
emoji?: string;
|
||||
children?: any;
|
||||
}
|
||||
|
||||
const { type = 'note', emoji } = Astro.props;
|
||||
|
||||
const typeConfig = {
|
||||
info: {
|
||||
bgColor: 'bg-blue-50 dark:bg-blue-900/20',
|
||||
borderColor: 'border-blue-200 dark:border-blue-800',
|
||||
textColor: 'text-blue-900 dark:text-blue-200',
|
||||
defaultEmoji: 'ℹ️'
|
||||
},
|
||||
warning: {
|
||||
bgColor: 'bg-yellow-50 dark:bg-yellow-900/20',
|
||||
borderColor: 'border-yellow-200 dark:border-yellow-800',
|
||||
textColor: 'text-yellow-900 dark:text-yellow-200',
|
||||
defaultEmoji: '⚠️'
|
||||
},
|
||||
error: {
|
||||
bgColor: 'bg-red-50 dark:bg-red-900/20',
|
||||
borderColor: 'border-red-200 dark:border-red-800',
|
||||
textColor: 'text-red-900 dark:text-red-200',
|
||||
defaultEmoji: '🚨'
|
||||
},
|
||||
success: {
|
||||
bgColor: 'bg-green-50 dark:bg-green-900/20',
|
||||
borderColor: 'border-green-200 dark:border-green-800',
|
||||
textColor: 'text-green-900 dark:text-green-200',
|
||||
defaultEmoji: '✅'
|
||||
},
|
||||
note: {
|
||||
bgColor: 'bg-gray-50 dark:bg-gray-900/20',
|
||||
borderColor: 'border-gray-200 dark:border-gray-800',
|
||||
textColor: 'text-gray-900 dark:text-gray-200',
|
||||
defaultEmoji: '📝'
|
||||
}
|
||||
};
|
||||
|
||||
const config = typeConfig[type] || typeConfig.note;
|
||||
const displayEmoji = emoji || config.defaultEmoji;
|
||||
---
|
||||
|
||||
<div class={`callout ${config.bgColor} ${config.borderColor} ${config.textColor} border rounded-lg p-4 my-4`}>
|
||||
<div class="flex items-start gap-3">
|
||||
<span class="text-xl flex-shrink-0" aria-hidden="true">{displayEmoji}</span>
|
||||
<div class="callout-content flex-1">
|
||||
<slot />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.callout {
|
||||
font-size: 0.9375rem;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.callout-content :global(p) {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.callout-content :global(p:not(:last-child)) {
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.callout-content :global(a) {
|
||||
text-decoration: underline;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.callout-content :global(code) {
|
||||
font-size: 0.875em;
|
||||
padding: 0.125rem 0.25rem;
|
||||
border-radius: 0.25rem;
|
||||
background-color: rgba(0, 0, 0, 0.05);
|
||||
}
|
||||
|
||||
:global(.dark) .callout-content :global(code) {
|
||||
background-color: rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
</style>
|
||||
@ -1,36 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
title: string;
|
||||
date: string;
|
||||
ogImage?: string;
|
||||
}
|
||||
|
||||
const { title, date, ogImage } = Astro.props;
|
||||
|
||||
// Format the date nicely
|
||||
const formattedDate = new Date(date).toLocaleDateString('en-US', {
|
||||
year: 'numeric',
|
||||
month: 'long',
|
||||
day: 'numeric'
|
||||
});
|
||||
---
|
||||
|
||||
<header class="changelog-header mb-8">
|
||||
{ogImage && (
|
||||
<div class="mt-6">
|
||||
<img
|
||||
src={ogImage}
|
||||
alt={title}
|
||||
class="w-full rounded-lg shadow-lg"
|
||||
loading="lazy"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</header>
|
||||
|
||||
<style>
|
||||
.changelog-header {
|
||||
border-bottom: 1px solid var(--sl-color-gray-5);
|
||||
padding-bottom: 2rem;
|
||||
}
|
||||
</style>
|
||||
@ -1,661 +0,0 @@
|
||||
---
|
||||
// Custom navigation component to add Products and API Reference links
|
||||
// This overrides the default Starlight Header component
|
||||
import Search from '@astrojs/starlight/components/Search.astro';
|
||||
import ThemeSelect from '@astrojs/starlight/components/ThemeSelect.astro';
|
||||
import { Icon } from '@astrojs/starlight/components';
|
||||
|
||||
// Determine if we're on a docs page based on the current path
|
||||
const currentPath = Astro.url.pathname;
|
||||
const isDocsPage = currentPath.startsWith('/jan/') ||
|
||||
currentPath.startsWith('/mobile/') ||
|
||||
currentPath.startsWith('/server/') ||
|
||||
currentPath.startsWith('/local-server/') ||
|
||||
currentPath === '/' ||
|
||||
currentPath === '/index' ||
|
||||
currentPath === '/docs' ||
|
||||
currentPath === '/docs/';
|
||||
---
|
||||
|
||||
<div class="sl-nav-wrapper">
|
||||
<nav class="sl-nav" aria-label="Main">
|
||||
<!-- Left side with title and links -->
|
||||
<div class="sl-nav__left">
|
||||
<!-- Site title/logo -->
|
||||
<a href="https://jan.ai" class="sl-nav__title">
|
||||
👋 Jan
|
||||
</a>
|
||||
|
||||
<!-- Main navigation links - hide on mobile for docs pages -->
|
||||
<div class={`sl-nav__links ${isDocsPage ? 'docs-page' : 'custom-page'}`}>
|
||||
<a href="/products" class="sl-nav__link" data-nav-item="products">
|
||||
Products
|
||||
</a>
|
||||
<a href="/jan" class="sl-nav__link" data-nav-item="docs">
|
||||
Docs
|
||||
</a>
|
||||
<a href="/handbook" class="sl-nav__link" data-nav-item="handbook">
|
||||
Handbook
|
||||
</a>
|
||||
<a href="/blog" class="sl-nav__link" data-nav-item="blog">
|
||||
Blog
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Center search -->
|
||||
<div class="sl-nav__center">
|
||||
<div class="sl-nav__search">
|
||||
<Search />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Right side items -->
|
||||
<div class="sl-nav__end">
|
||||
<!-- Changelog and API Reference links -->
|
||||
<a href="/changelog" class={`sl-nav__link sl-nav__api-link ${isDocsPage ? 'docs-page' : 'custom-page'}`} data-nav-item="changelog">
|
||||
Changelog
|
||||
</a>
|
||||
<a href="/api-reference" class={`sl-nav__link sl-nav__api-link ${isDocsPage ? 'docs-page' : 'custom-page'}`} data-nav-item="api">
|
||||
API Reference
|
||||
</a>
|
||||
|
||||
<!-- Theme toggle - always visible on desktop -->
|
||||
<div class="sl-nav__theme">
|
||||
<ThemeSelect />
|
||||
</div>
|
||||
|
||||
<!-- Social links -->
|
||||
<div class="sl-nav__social">
|
||||
<a href="https://github.com/menloresearch/jan" class="sl-nav__social-link" aria-label="GitHub">
|
||||
<Icon name="github"/>
|
||||
</a>
|
||||
<a href="https://twitter.com/jandotai" class="sl-nav__social-link" aria-label="X">
|
||||
<Icon name="x.com"/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/FTk2MvZwJH" class="sl-nav__social-link" aria-label="Discord">
|
||||
<Icon name="discord"/>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- Mobile hamburger menu - only for custom pages -->
|
||||
{!isDocsPage && (
|
||||
<div class="sl-nav__hamburger">
|
||||
<button id="hamburger-btn" class="hamburger-button" aria-label="Toggle navigation menu">
|
||||
<span class="hamburger-icon">☰</span>
|
||||
</button>
|
||||
<div id="hamburger-menu" class="hamburger-menu">
|
||||
<a href="/products" class="hamburger-link">Products</a>
|
||||
<a href="/jan" class="hamburger-link">Docs</a>
|
||||
<a href="/handbook" class="hamburger-link">Handbook</a>
|
||||
<a href="/blog" class="hamburger-link">Blog</a>
|
||||
<a href="/changelog" class="hamburger-link">Changelog</a>
|
||||
<a href="/api-reference" class="hamburger-link">API Reference</a>
|
||||
<div class="hamburger-social">
|
||||
<a href="https://github.com/menloresearch/jan" class="hamburger-social-link" aria-label="GitHub">
|
||||
<Icon name="github"/>
|
||||
</a>
|
||||
<a href="https://twitter.com/jandotai" class="hamburger-social-link" aria-label="X">
|
||||
<Icon name="x.com"/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/FTk2MvZwJH" class="hamburger-social-link" aria-label="Discord">
|
||||
<Icon name="discord"/>
|
||||
</a>
|
||||
</div>
|
||||
<div class="hamburger-theme">
|
||||
<ThemeSelect />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
/* Base navigation styles */
|
||||
.sl-nav-wrapper {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 999; /* High z-index to stay above ToC and other elements */
|
||||
background: var(--sl-color-bg-nav);
|
||||
border-bottom: 1px solid var(--sl-color-hairline);
|
||||
backdrop-filter: blur(10px);
|
||||
-webkit-backdrop-filter: blur(10px);
|
||||
}
|
||||
|
||||
.sl-nav {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
padding: 0.75rem 1rem;
|
||||
max-width: 100%;
|
||||
margin: 0 auto;
|
||||
height: var(--sl-nav-height, 3.5rem);
|
||||
}
|
||||
|
||||
.sl-nav__center {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
max-width: 600px;
|
||||
margin: 0 1rem;
|
||||
}
|
||||
|
||||
.sl-nav__left {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.sl-nav__title {
|
||||
font-size: 1.125rem;
|
||||
font-weight: 600;
|
||||
color: var(--sl-color-white);
|
||||
text-decoration: none;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.sl-nav__title:hover {
|
||||
color: var(--sl-color-text-accent);
|
||||
}
|
||||
|
||||
.sl-nav__links {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.sl-nav__link {
|
||||
color: var(--sl-color-gray-3);
|
||||
text-decoration: none;
|
||||
font-size: 0.9375rem;
|
||||
transition: color 0.2s ease;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.sl-nav__link:hover {
|
||||
color: var(--sl-color-white);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
/* Light theme adjustments - better contrast */
|
||||
html[data-theme="light"] .sl-nav__link {
|
||||
color: #374151; /* Darker gray for better readability */
|
||||
}
|
||||
|
||||
html[data-theme="light"] .sl-nav__link:hover {
|
||||
color: var(--sl-color-accent);
|
||||
}
|
||||
|
||||
html[data-theme="light"] .sl-nav__title {
|
||||
color: #111827; /* Very dark gray, almost black */
|
||||
}
|
||||
|
||||
/* Active link styling */
|
||||
.sl-nav__link.active {
|
||||
color: var(--sl-color-text-accent);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Light theme active link */
|
||||
html[data-theme="light"] .sl-nav__link.active {
|
||||
color: var(--sl-color-text-accent);
|
||||
}
|
||||
|
||||
.sl-nav__end {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.sl-nav__search {
|
||||
width: 100%;
|
||||
max-width: 400px;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
/* Fix search input styling for light mode */
|
||||
.sl-nav__search :global(input) {
|
||||
width: 100%;
|
||||
padding: 0.5rem 1rem;
|
||||
}
|
||||
|
||||
/* Fix search bar in light mode with more specific selectors */
|
||||
html[data-theme="light"] .sl-nav__search :global(.pagefind-ui__search-input),
|
||||
html[data-theme="light"] .sl-nav__search :global(input[type="search"]),
|
||||
html[data-theme="light"] .sl-nav__search :global(input) {
|
||||
background: #f9fafb !important;
|
||||
background-color: #f9fafb !important;
|
||||
border: 1px solid #d1d5db !important;
|
||||
color: #111827 !important;
|
||||
}
|
||||
|
||||
html[data-theme="light"] .sl-nav__search :global(.pagefind-ui__search-input:focus),
|
||||
html[data-theme="light"] .sl-nav__search :global(input[type="search"]:focus),
|
||||
html[data-theme="light"] .sl-nav__search :global(input:focus) {
|
||||
background: #ffffff !important;
|
||||
background-color: #ffffff !important;
|
||||
border-color: var(--sl-color-accent) !important;
|
||||
color: #111827 !important;
|
||||
}
|
||||
|
||||
html[data-theme="light"] .sl-nav__search :global(.pagefind-ui__search-input::placeholder),
|
||||
html[data-theme="light"] .sl-nav__search :global(input::placeholder) {
|
||||
color: #9ca3af !important;
|
||||
}
|
||||
|
||||
.sl-nav__social {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.sl-nav__social-link {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 0.375rem;
|
||||
color: var(--sl-color-gray-3);
|
||||
transition: color 0.2s ease;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.sl-nav__social-link:hover {
|
||||
color: var(--sl-color-white);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
/* Light theme social links - better contrast */
|
||||
html[data-theme="light"] .sl-nav__social-link {
|
||||
color: #6b7280; /* Medium gray for icons */
|
||||
}
|
||||
|
||||
html[data-theme="light"] .sl-nav__social-link:hover {
|
||||
color: var(--sl-color-accent);
|
||||
}
|
||||
|
||||
/* Theme selector container */
|
||||
.sl-nav__theme {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
/* Hide only the "Select theme" label text, keep dropdown visible */
|
||||
.sl-nav__theme :global(starlight-theme-select) {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
/* Hide only the screen reader text */
|
||||
.sl-nav__theme :global(starlight-theme-select .sr-only) {
|
||||
position: absolute !important;
|
||||
width: 1px !important;
|
||||
height: 1px !important;
|
||||
padding: 0 !important;
|
||||
margin: -1px !important;
|
||||
overflow: hidden !important;
|
||||
clip: rect(0, 0, 0, 0) !important;
|
||||
white-space: nowrap !important;
|
||||
border: 0 !important;
|
||||
}
|
||||
|
||||
/* Style the select dropdown with visible text */
|
||||
.sl-nav__theme :global(starlight-theme-select select) {
|
||||
font-size: 0.875rem !important;
|
||||
color: var(--sl-color-text) !important;
|
||||
background: transparent !important;
|
||||
border: none !important;
|
||||
cursor: pointer !important;
|
||||
padding-right: 1.5rem !important;
|
||||
}
|
||||
|
||||
/* Ensure options are visible with proper contrast */
|
||||
.sl-nav__theme :global(starlight-theme-select option) {
|
||||
font-size: 0.875rem !important;
|
||||
color: var(--sl-color-black) !important;
|
||||
background: var(--sl-color-white) !important;
|
||||
}
|
||||
|
||||
/* Light mode select text color */
|
||||
html[data-theme="light"] .sl-nav__theme :global(starlight-theme-select select) {
|
||||
color: var(--sl-color-gray-5) !important;
|
||||
}
|
||||
|
||||
/* Hide the icon to prevent overlap with text */
|
||||
.sl-nav__theme :global(starlight-theme-select svg) {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Hamburger menu styles */
|
||||
.sl-nav__hamburger {
|
||||
display: none;
|
||||
position: relative; /* Changed from static to contain the menu properly */
|
||||
}
|
||||
|
||||
.hamburger-button {
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: var(--sl-color-white);
|
||||
font-size: 1.5rem;
|
||||
cursor: pointer;
|
||||
padding: 0.375rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
transition: color 0.2s ease;
|
||||
}
|
||||
|
||||
/* Fix hamburger button visibility in light mode */
|
||||
html[data-theme="light"] .hamburger-button {
|
||||
color: #374151; /* Darker gray for better visibility */
|
||||
}
|
||||
|
||||
.hamburger-button:hover {
|
||||
color: var(--sl-color-text-accent);
|
||||
}
|
||||
|
||||
html[data-theme="light"] .hamburger-button:hover {
|
||||
color: var(--sl-color-text-accent);
|
||||
}
|
||||
|
||||
.hamburger-menu {
|
||||
display: none;
|
||||
position: absolute;
|
||||
top: calc(100% + 0.5rem); /* Add small gap below navbar */
|
||||
right: 0;
|
||||
background: var(--sl-color-bg-nav);
|
||||
border: 1px solid var(--sl-color-hairline);
|
||||
border-radius: 0.5rem;
|
||||
padding: 1rem;
|
||||
min-width: 200px;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
|
||||
z-index: 1000; /* Ensure menu stays above other content */
|
||||
}
|
||||
|
||||
/* Light mode menu background */
|
||||
html[data-theme="light"] .hamburger-menu {
|
||||
background: var(--sl-color-bg);
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.hamburger-menu.show {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.hamburger-link {
|
||||
display: block;
|
||||
padding: 0.5rem 1rem;
|
||||
color: var(--sl-color-gray-3);
|
||||
text-decoration: none;
|
||||
transition: color 0.2s ease, background 0.2s ease;
|
||||
border-radius: 0.25rem;
|
||||
}
|
||||
|
||||
/* Light mode hamburger links */
|
||||
html[data-theme="light"] .hamburger-link {
|
||||
color: #374151; /* Consistent with nav links */
|
||||
}
|
||||
|
||||
.hamburger-link:hover {
|
||||
color: var(--sl-color-white);
|
||||
background: var(--sl-color-gray-6);
|
||||
}
|
||||
|
||||
html[data-theme="light"] .hamburger-link:hover {
|
||||
color: var(--sl-color-accent);
|
||||
background: var(--sl-color-gray-6);
|
||||
}
|
||||
|
||||
.hamburger-social {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
padding: 0.5rem 1rem;
|
||||
margin-top: 0.5rem;
|
||||
border-top: 1px solid var(--sl-color-hairline);
|
||||
}
|
||||
|
||||
.hamburger-social-link {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 0.375rem;
|
||||
color: var(--sl-color-gray-3);
|
||||
transition: color 0.2s ease;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
html[data-theme="light"] .hamburger-social-link {
|
||||
color: #6b7280; /* Consistent with nav social links */
|
||||
}
|
||||
|
||||
.hamburger-social-link:hover {
|
||||
color: var(--sl-color-white);
|
||||
}
|
||||
|
||||
html[data-theme="light"] .hamburger-social-link:hover {
|
||||
color: var(--sl-color-accent);
|
||||
}
|
||||
|
||||
.hamburger-theme {
|
||||
padding: 0.5rem 1rem;
|
||||
margin-top: 0.5rem;
|
||||
border-top: 1px solid var(--sl-color-hairline);
|
||||
}
|
||||
|
||||
/* Search component integration styles */
|
||||
.sl-nav__search :global(.pagefind-ui) {
|
||||
--pagefind-ui-scale: 0.9;
|
||||
}
|
||||
|
||||
.sl-nav__search :global(.pagefind-ui__search-input) {
|
||||
background: var(--sl-color-gray-6);
|
||||
border: 1px solid var(--sl-color-gray-5);
|
||||
color: var(--sl-color-white);
|
||||
border-radius: 0.375rem;
|
||||
font-size: 0.875rem;
|
||||
padding: 0.375rem 1rem;
|
||||
transition: border-color 0.2s ease, background 0.2s ease;
|
||||
}
|
||||
|
||||
/* Override any Starlight defaults for search in dark mode */
|
||||
html[data-theme="dark"] .sl-nav__search :global(.pagefind-ui__search-input),
|
||||
html[data-theme="dark"] .sl-nav__search :global(input[type="search"]),
|
||||
html[data-theme="dark"] .sl-nav__search :global(input) {
|
||||
background: #1f2937 !important;
|
||||
background-color: #1f2937 !important;
|
||||
border: 1px solid #374151 !important;
|
||||
color: #f3f4f6 !important;
|
||||
}
|
||||
|
||||
.sl-nav__search :global(.pagefind-ui__search-input:hover) {
|
||||
border-color: var(--sl-color-gray-4);
|
||||
background: var(--sl-color-gray-5);
|
||||
}
|
||||
|
||||
.sl-nav__search :global(.pagefind-ui__search-input:focus) {
|
||||
outline: none;
|
||||
border-color: var(--sl-color-text-accent);
|
||||
background: var(--sl-color-bg);
|
||||
}
|
||||
|
||||
.sl-nav__search :global(.pagefind-ui__search-input::placeholder) {
|
||||
color: var(--sl-color-gray-3);
|
||||
}
|
||||
|
||||
/* Responsive styles */
|
||||
|
||||
/* Tablet view */
|
||||
@media (max-width: 900px) {
|
||||
.sl-nav__links {
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.sl-nav__link {
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.sl-nav__social {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
/* Mobile view - differentiate between docs and custom pages */
|
||||
@media (max-width: 768px) {
|
||||
.sl-nav {
|
||||
padding: 0.5rem 1rem;
|
||||
}
|
||||
|
||||
.sl-nav__center {
|
||||
margin: 0 0.5rem;
|
||||
}
|
||||
|
||||
/* For docs pages - hide custom nav items on mobile, let Starlight handle it */
|
||||
.sl-nav__links.docs-page {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.sl-nav__api-link.docs-page {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* For custom pages - hide nav items but show hamburger */
|
||||
.sl-nav__links.custom-page {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.sl-nav__api-link.custom-page {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.sl-nav__theme {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Show hamburger only for custom pages */
|
||||
.sl-nav__hamburger {
|
||||
display: block;
|
||||
}
|
||||
}
|
||||
|
||||
/* Small mobile */
|
||||
@media (max-width: 640px) {
|
||||
.sl-nav {
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.sl-nav__left {
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.sl-nav__center {
|
||||
flex: 1;
|
||||
margin: 0 0.5rem;
|
||||
}
|
||||
|
||||
.sl-nav__end {
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.sl-nav__title {
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.sl-nav__search :global(.pagefind-ui__search-input) {
|
||||
padding: 0.25rem 0.75rem;
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
}
|
||||
|
||||
/* Make sure content doesn't jump */
|
||||
:global(body) {
|
||||
padding-top: 0;
|
||||
}
|
||||
|
||||
:global(.sl-layout) {
|
||||
padding-top: 0;
|
||||
}
|
||||
|
||||
/* Adjust main content positioning */
|
||||
:global(main.sl-content) {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
:global(.sl-content__inner) {
|
||||
padding-top: 1rem;
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
// Add active state highlighting based on current page
|
||||
const currentPath = window.location.pathname;
|
||||
const links = document.querySelectorAll('.sl-nav__link');
|
||||
|
||||
links.forEach(link => {
|
||||
const href = link.getAttribute('href');
|
||||
if (href) {
|
||||
// Check for exact match or if current path starts with the link href
|
||||
if (currentPath === href ||
|
||||
(href !== '/' && currentPath.startsWith(href))) {
|
||||
link.classList.add('active');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Theme handling for custom pages
|
||||
// Listen for theme changes from Starlight's ThemeSelect
|
||||
const observeThemeChanges = () => {
|
||||
const observer = new MutationObserver((mutations) => {
|
||||
mutations.forEach((mutation) => {
|
||||
if (mutation.type === 'attributes' && mutation.attributeName === 'data-theme') {
|
||||
const newTheme = document.documentElement.getAttribute('data-theme');
|
||||
// Theme is already set by Starlight, we just need to ensure it persists
|
||||
localStorage.setItem('starlight-theme', newTheme === 'dark' ? 'dark' : 'light');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['data-theme']
|
||||
});
|
||||
};
|
||||
|
||||
observeThemeChanges();
|
||||
|
||||
// Hamburger menu functionality (only for custom pages)
|
||||
const hamburgerBtn = document.getElementById('hamburger-btn');
|
||||
const hamburgerMenu = document.getElementById('hamburger-menu');
|
||||
|
||||
if (hamburgerBtn && hamburgerMenu) {
|
||||
// Toggle menu on button click
|
||||
hamburgerBtn.addEventListener('click', function(e) {
|
||||
e.stopPropagation();
|
||||
hamburgerMenu.classList.toggle('show');
|
||||
});
|
||||
|
||||
// Close menu when clicking outside
|
||||
document.addEventListener('click', function(e) {
|
||||
if (!hamburgerBtn.contains(e.target) && !hamburgerMenu.contains(e.target)) {
|
||||
hamburgerMenu.classList.remove('show');
|
||||
}
|
||||
});
|
||||
|
||||
// Close menu when clicking a link
|
||||
const hamburgerLinks = hamburgerMenu.querySelectorAll('.hamburger-link');
|
||||
hamburgerLinks.forEach(link => {
|
||||
link.addEventListener('click', function() {
|
||||
hamburgerMenu.classList.remove('show');
|
||||
});
|
||||
});
|
||||
|
||||
// Close menu on escape key
|
||||
document.addEventListener('keydown', function(e) {
|
||||
if (e.key === 'Escape' && hamburgerMenu.classList.contains('show')) {
|
||||
hamburgerMenu.classList.remove('show');
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
</script>
|
||||
@ -1,233 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
class?: string;
|
||||
showStats?: boolean;
|
||||
downloadCount?: string;
|
||||
}
|
||||
|
||||
const { class: className, showStats = false, downloadCount = '3.8M+' } = Astro.props;
|
||||
|
||||
// Download links for different platforms
|
||||
const downloadLinks = {
|
||||
'mac-intel': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-mac-x64-0.5.14.dmg',
|
||||
'mac-arm': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-mac-arm64-0.5.14.dmg',
|
||||
'windows': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-win-x64-0.5.14.exe',
|
||||
'linux-deb': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-linux-amd64-0.5.14.deb',
|
||||
'linux-appimage': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-linux-x86_64-0.5.14.AppImage'
|
||||
};
|
||||
---
|
||||
|
||||
<div class={`download-container ${className || ''}`}>
|
||||
<div class="download-section inline-flex items-center gap-2">
|
||||
<!-- Main Download Button -->
|
||||
<button
|
||||
id="main-download-btn"
|
||||
class="download-btn inline-flex items-center gap-2 px-5 py-2.5 bg-black dark:bg-white text-white dark:text-black rounded-lg hover:opacity-90 transition-all duration-200 font-medium shadow-sm hover:shadow-md"
|
||||
data-download-url=""
|
||||
>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M9 19l3 3m0 0l3-3m-3 3V10" />
|
||||
</svg>
|
||||
<span id="platform-text" class="text-base">Download for your platform</span>
|
||||
<span id="file-size" class="text-xs opacity-70"></span>
|
||||
</button>
|
||||
|
||||
<!-- Dropdown Toggle -->
|
||||
<details class="download-dropdown relative">
|
||||
<summary class="dropdown-toggle cursor-pointer p-2.5 bg-gray-100 dark:bg-gray-800 rounded-lg hover:bg-gray-200 dark:hover:bg-gray-700 transition-colors h-full flex items-center">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7" />
|
||||
</svg>
|
||||
</summary>
|
||||
|
||||
<!-- Dropdown Menu -->
|
||||
<div class="dropdown-menu absolute top-full mt-1 right-0 bg-white dark:bg-gray-800 rounded-lg shadow-lg border border-gray-200 dark:border-gray-700 p-1 min-w-[250px] z-50">
|
||||
<div class="font-semibold text-xs text-gray-500 dark:text-gray-400 px-3 py-1.5 border-b border-gray-200 dark:border-gray-700">
|
||||
All Platforms
|
||||
</div>
|
||||
|
||||
<!-- macOS Options -->
|
||||
<div class="py-1">
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400 px-3 py-0.5 uppercase tracking-wide">macOS</div>
|
||||
<a href={downloadLinks['mac-arm']}
|
||||
class="platform-option block px-3 py-1.5 hover:bg-gray-100 dark:hover:bg-gray-700 rounded transition-colors text-sm"
|
||||
data-platform="mac-arm">
|
||||
<span class="font-medium">Apple Silicon</span>
|
||||
<span class="text-sm text-gray-500 dark:text-gray-400 ml-2">(M1/M2/M3)</span>
|
||||
</a>
|
||||
<a href={downloadLinks['mac-intel']}
|
||||
class="platform-option block px-3 py-1.5 hover:bg-gray-100 dark:hover:bg-gray-700 rounded transition-colors text-sm"
|
||||
data-platform="mac-intel">
|
||||
<span class="font-medium">Intel</span>
|
||||
<span class="text-sm text-gray-500 dark:text-gray-400 ml-2">(x64)</span>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- Windows -->
|
||||
<div class="py-1 border-t border-gray-200 dark:border-gray-700">
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400 px-3 py-0.5 uppercase tracking-wide">Windows</div>
|
||||
<a href={downloadLinks['windows']}
|
||||
class="platform-option block px-3 py-1.5 hover:bg-gray-100 dark:hover:bg-gray-700 rounded transition-colors text-sm"
|
||||
data-platform="windows">
|
||||
<span class="font-medium">Windows</span>
|
||||
<span class="text-sm text-gray-500 dark:text-gray-400 ml-2">(x64)</span>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- Linux Options -->
|
||||
<div class="py-1 border-t border-gray-200 dark:border-gray-700">
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400 px-3 py-0.5 uppercase tracking-wide">Linux</div>
|
||||
<a href={downloadLinks['linux-deb']}
|
||||
class="platform-option block px-3 py-1.5 hover:bg-gray-100 dark:hover:bg-gray-700 rounded transition-colors text-sm"
|
||||
data-platform="linux-deb">
|
||||
<span class="font-medium">Debian/Ubuntu</span>
|
||||
<span class="text-sm text-gray-500 dark:text-gray-400 ml-2">(.deb)</span>
|
||||
</a>
|
||||
<a href={downloadLinks['linux-appimage']}
|
||||
class="platform-option block px-3 py-1.5 hover:bg-gray-100 dark:hover:bg-gray-700 rounded transition-colors text-sm"
|
||||
data-platform="linux-appimage">
|
||||
<span class="font-medium">AppImage</span>
|
||||
<span class="text-sm text-gray-500 dark:text-gray-400 ml-2">(Universal)</span>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- View All Releases -->
|
||||
<div class="pt-1 border-t border-gray-200 dark:border-gray-700">
|
||||
<a href="https://github.com/janhq/jan/releases"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="block px-3 py-1.5 text-blue-600 dark:text-blue-400 hover:bg-gray-100 dark:hover:bg-gray-700 rounded transition-colors text-xs">
|
||||
View all releases →
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
{showStats && (
|
||||
<p class="text-gray-600 dark:text-gray-400 mt-3 text-center text-sm">
|
||||
<span class="text-yellow-600 font-semibold">{downloadCount}</span> downloads | Free & Open Source
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.download-dropdown[open] .dropdown-menu {
|
||||
animation: slideDown 0.2s ease-out;
|
||||
}
|
||||
|
||||
.download-dropdown summary::-webkit-details-marker {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.download-dropdown summary {
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.download-btn:active {
|
||||
transform: scale(0.98);
|
||||
}
|
||||
|
||||
@keyframes slideDown {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-10px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
/* Close dropdown when clicking outside */
|
||||
.download-dropdown:not([open]) .dropdown-menu {
|
||||
display: none;
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
// Platform detection and download handling
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const platformText = document.getElementById('platform-text');
|
||||
const fileSize = document.getElementById('file-size');
|
||||
const mainDownloadBtn = document.getElementById('main-download-btn');
|
||||
|
||||
// Platform detection
|
||||
function detectPlatform() {
|
||||
const platform = navigator.platform.toLowerCase();
|
||||
const userAgent = navigator.userAgent.toLowerCase();
|
||||
|
||||
let detectedPlatform = 'linux-deb';
|
||||
let platformName = 'Linux (deb)';
|
||||
let downloadSize = '(69.7 MB)';
|
||||
|
||||
if (platform.includes('mac') || userAgent.includes('mac')) {
|
||||
// Check if Apple Silicon
|
||||
if (userAgent.includes('arm') || userAgent.includes('apple')) {
|
||||
detectedPlatform = 'mac-arm';
|
||||
platformName = 'macOS (Apple Silicon)';
|
||||
downloadSize = '(73.2 MB)';
|
||||
} else {
|
||||
detectedPlatform = 'mac-intel';
|
||||
platformName = 'macOS (Intel)';
|
||||
downloadSize = '(75.8 MB)';
|
||||
}
|
||||
} else if (platform.includes('win') || userAgent.includes('win')) {
|
||||
detectedPlatform = 'windows';
|
||||
platformName = 'Windows';
|
||||
downloadSize = '(71.4 MB)';
|
||||
}
|
||||
|
||||
return { platform: detectedPlatform, name: platformName, size: downloadSize };
|
||||
}
|
||||
|
||||
// Set initial platform
|
||||
const detected = detectPlatform();
|
||||
if (platformText) {
|
||||
platformText.textContent = `Download for ${detected.name}`;
|
||||
}
|
||||
if (fileSize) {
|
||||
fileSize.textContent = detected.size;
|
||||
}
|
||||
|
||||
// Get download URL based on platform
|
||||
const downloadUrls = {
|
||||
'mac-intel': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-mac-x64-0.5.14.dmg',
|
||||
'mac-arm': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-mac-arm64-0.5.14.dmg',
|
||||
'windows': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-win-x64-0.5.14.exe',
|
||||
'linux-deb': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-linux-amd64-0.5.14.deb',
|
||||
'linux-appimage': 'https://github.com/janhq/jan/releases/download/v0.5.14/jan-linux-x86_64-0.5.14.AppImage'
|
||||
};
|
||||
|
||||
if (mainDownloadBtn) {
|
||||
mainDownloadBtn.setAttribute('data-download-url', downloadUrls[detected.platform]);
|
||||
|
||||
// Handle main button click
|
||||
mainDownloadBtn.addEventListener('click', () => {
|
||||
const url = mainDownloadBtn.getAttribute('data-download-url');
|
||||
if (url) {
|
||||
window.location.href = url;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Handle dropdown platform clicks
|
||||
document.querySelectorAll('.platform-option').forEach(link => {
|
||||
link.addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
const url = e.currentTarget.getAttribute('href');
|
||||
if (url) {
|
||||
window.location.href = url;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Close dropdown when clicking outside
|
||||
document.addEventListener('click', (e) => {
|
||||
const dropdown = document.querySelector('.download-dropdown');
|
||||
if (dropdown && !dropdown.contains(e.target)) {
|
||||
dropdown.removeAttribute('open');
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
@ -1,671 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const { className = '' } = Astro.props;
|
||||
---
|
||||
|
||||
<div class={`release-database ${className}`}>
|
||||
<table class="newspaper-table">
|
||||
<caption>Release Database</caption>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Release Version</th>
|
||||
<th>Target Date</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<!-- v0.6.7 - Released -->
|
||||
<tr class="release-row" data-version="v0.6.7">
|
||||
<td class="version-cell">
|
||||
<strong>v0.6.7</strong>
|
||||
<span class="version-name">Web Search</span>
|
||||
</td>
|
||||
<td>Released</td>
|
||||
<td><span class="status released">Live</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v0.6.7">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>User Stories</h5>
|
||||
<ul>
|
||||
<li>As a user, I want to search the web directly from Jan chat for real-time information</li>
|
||||
<li>As a developer, I want better API documentation for integration</li>
|
||||
<li>As a researcher, I want source attribution for all search results</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>New Features</h5>
|
||||
<ul>
|
||||
<li>Privacy-respecting web search integration</li>
|
||||
<li>Multiple search engine support</li>
|
||||
<li>Improved model downloading experience</li>
|
||||
<li>Enhanced error handling and user feedback</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Documentation</h5>
|
||||
<ul>
|
||||
<li>Web Search API documentation</li>
|
||||
<li>Privacy policy updates</li>
|
||||
<li>Search configuration guide</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- v0.6.5 - Released -->
|
||||
<tr class="release-row" data-version="v0.6.5">
|
||||
<td class="version-cell">
|
||||
<strong>v0.6.5</strong>
|
||||
<span class="version-name">Desktop Stability</span>
|
||||
</td>
|
||||
<td>Released</td>
|
||||
<td><span class="status released">Live</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v0.6.5">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>Improvements</h5>
|
||||
<ul>
|
||||
<li>Enhanced memory management for large models</li>
|
||||
<li>Improved GPU utilization</li>
|
||||
<li>Better cross-platform compatibility</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Bug Fixes</h5>
|
||||
<ul>
|
||||
<li>Fixed model loading crashes on Windows</li>
|
||||
<li>Resolved memory leaks in long conversations</li>
|
||||
<li>Fixed UI freezing during model downloads</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Performance</h5>
|
||||
<ul>
|
||||
<li>30% faster model loading times</li>
|
||||
<li>Reduced memory footprint by 25%</li>
|
||||
<li>Improved response streaming</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- v0.7.0 - In Development -->
|
||||
<tr class="release-row" data-version="v0.7.0">
|
||||
<td class="version-cell">
|
||||
<strong>v0.7.0</strong>
|
||||
<span class="version-name">Deep Research</span>
|
||||
</td>
|
||||
<td>January 2025</td>
|
||||
<td><span class="status development">Building</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v0.7.0">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>User Stories</h5>
|
||||
<ul>
|
||||
<li>As a researcher, I want to conduct deep research with multiple sources and citations</li>
|
||||
<li>As a user, I want Jan Nano to work seamlessly on mobile devices</li>
|
||||
<li>As a team lead, I want to preview jan.ai for my organization</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>New Features</h5>
|
||||
<ul>
|
||||
<li>Deep research capabilities with multi-source analysis</li>
|
||||
<li>jan.ai web platform beta launch</li>
|
||||
<li>Improved Jan Nano performance and mobile optimization</li>
|
||||
<li>Citation management and research reports</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Documentation</h5>
|
||||
<ul>
|
||||
<li>Research methodology guide</li>
|
||||
<li>jan.ai platform documentation</li>
|
||||
<li>Mobile optimization best practices</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- v0.8.0 - In Development -->
|
||||
<tr class="release-row" data-version="v0.8.0">
|
||||
<td class="version-cell">
|
||||
<strong>v0.8.0</strong>
|
||||
<span class="version-name">Browser Automation</span>
|
||||
</td>
|
||||
<td>March 2025</td>
|
||||
<td><span class="status development">Building</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v0.8.0">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>User Stories</h5>
|
||||
<ul>
|
||||
<li>As a user, I want Jan to automate web tasks for me safely and efficiently</li>
|
||||
<li>As a business user, I want to automate repetitive web workflows</li>
|
||||
<li>As a developer, I want browser automation APIs for custom integrations</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>New Features</h5>
|
||||
<ul>
|
||||
<li>Browser automation capabilities (Beta)</li>
|
||||
<li>Form filling and data extraction</li>
|
||||
<li>Website navigation and interaction</li>
|
||||
<li>Safety policies and user confirmation flows</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Documentation</h5>
|
||||
<ul>
|
||||
<li>Browser automation safety guide</li>
|
||||
<li>Web scraping best practices</li>
|
||||
<li>API reference for automation tools</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- v0.9.0 - Planned -->
|
||||
<tr class="release-row" data-version="v0.9.0">
|
||||
<td class="version-cell">
|
||||
<strong>v0.9.0</strong>
|
||||
<span class="version-name">Mobile Launch</span>
|
||||
</td>
|
||||
<td>Q1 2025</td>
|
||||
<td><span class="status planned">Planned</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v0.9.0">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>User Stories</h5>
|
||||
<ul>
|
||||
<li>As a mobile user, I want Jan AI available on my iOS/Android device</li>
|
||||
<li>As a commuter, I want to seamlessly switch between desktop and mobile</li>
|
||||
<li>As a privacy-conscious user, I want local AI on my phone</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>New Features</h5>
|
||||
<ul>
|
||||
<li>iOS and Android mobile applications</li>
|
||||
<li>Three adaptive modes (Desktop, Server, Local)</li>
|
||||
<li>Voice-first interface</li>
|
||||
<li>Cross-device synchronization</li>
|
||||
<li>Jan Nano on-device processing</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Documentation</h5>
|
||||
<ul>
|
||||
<li>Mobile setup and configuration guide</li>
|
||||
<li>Voice interaction best practices</li>
|
||||
<li>Device syncing tutorial</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- v1.0.0 - Planned -->
|
||||
<tr class="release-row" data-version="v1.0.0">
|
||||
<td class="version-cell">
|
||||
<strong>v1.0.0</strong>
|
||||
<span class="version-name">Server Edition</span>
|
||||
</td>
|
||||
<td>Q2 2025</td>
|
||||
<td><span class="status planned">Planned</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v1.0.0">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>User Stories</h5>
|
||||
<ul>
|
||||
<li>As an IT admin, I want to deploy Jan for my entire organization</li>
|
||||
<li>As a team lead, I want centralized AI with team collaboration features</li>
|
||||
<li>As a CTO, I want enterprise-grade security and compliance</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>New Features</h5>
|
||||
<ul>
|
||||
<li>Multi-user server deployment</li>
|
||||
<li>Enterprise authentication (SSO, LDAP)</li>
|
||||
<li>Team collaboration and shared conversations</li>
|
||||
<li>Docker and Kubernetes deployment options</li>
|
||||
<li>Admin dashboard and user management</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Documentation</h5>
|
||||
<ul>
|
||||
<li>Enterprise deployment guide</li>
|
||||
<li>Security and compliance documentation</li>
|
||||
<li>Admin management tutorials</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- v1.1.0 - Planned -->
|
||||
<tr class="release-row" data-version="v1.1.0">
|
||||
<td class="version-cell">
|
||||
<strong>v1.1.0</strong>
|
||||
<span class="version-name">AI Agents</span>
|
||||
</td>
|
||||
<td>Q1 2026</td>
|
||||
<td><span class="status planned">Planned</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v1.1.0">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>User Stories</h5>
|
||||
<ul>
|
||||
<li>As a user, I want autonomous agents to handle complex multi-step tasks</li>
|
||||
<li>As a business owner, I want AI that can coordinate multiple tools automatically</li>
|
||||
<li>As a power user, I want to create custom agent workflows</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>New Features</h5>
|
||||
<ul>
|
||||
<li>Autonomous AI agents</li>
|
||||
<li>Multi-step task planning and execution</li>
|
||||
<li>Tool orchestration and coordination</li>
|
||||
<li>Goal-oriented intelligent reasoning</li>
|
||||
<li>Custom agent workflow builder</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Documentation</h5>
|
||||
<ul>
|
||||
<li>Agent development guide</li>
|
||||
<li>Workflow automation tutorials</li>
|
||||
<li>Safety and oversight best practices</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- v1.2.0 - Planned -->
|
||||
<tr class="release-row" data-version="v1.2.0">
|
||||
<td class="version-cell">
|
||||
<strong>v1.2.0</strong>
|
||||
<span class="version-name">Lucy Multimodal</span>
|
||||
</td>
|
||||
<td>Q2 2025</td>
|
||||
<td><span class="status planned">Planned</span></td>
|
||||
</tr>
|
||||
<tr class="release-details" data-for="v1.2.0">
|
||||
<td colspan="3">
|
||||
<div class="details-grid">
|
||||
<div class="detail-section">
|
||||
<h5>User Stories</h5>
|
||||
<ul>
|
||||
<li>As a content creator, I want AI that can understand and process images</li>
|
||||
<li>As a researcher, I want to analyze documents with both text and visuals</li>
|
||||
<li>As a designer, I want AI that can help with visual content creation</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>New Features</h5>
|
||||
<ul>
|
||||
<li>Lucy multimodal model release</li>
|
||||
<li>Image understanding and analysis</li>
|
||||
<li>Document processing with OCR</li>
|
||||
<li>Visual reasoning capabilities</li>
|
||||
<li>Audio processing (planned)</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
<h5>Documentation</h5>
|
||||
<ul>
|
||||
<li>Multimodal AI usage guide</li>
|
||||
<li>Image processing tutorials</li>
|
||||
<li>Visual reasoning examples</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.release-database {
|
||||
margin: 3rem 0;
|
||||
}
|
||||
|
||||
/* Newspaper-style table */
|
||||
.newspaper-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 0.9rem;
|
||||
font-family: Georgia, 'Times New Roman', serif;
|
||||
margin: 2rem 0;
|
||||
}
|
||||
|
||||
.newspaper-table caption {
|
||||
font-size: 1.4rem;
|
||||
font-weight: 700;
|
||||
margin-bottom: 1rem;
|
||||
text-align: left;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
}
|
||||
|
||||
.newspaper-table th,
|
||||
.newspaper-table td {
|
||||
padding: 0.75rem;
|
||||
text-align: left;
|
||||
border-bottom: 1px solid #000;
|
||||
}
|
||||
|
||||
.newspaper-table th {
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
background: #f0f0f0;
|
||||
border-top: 3px double #000;
|
||||
border-bottom: 1px solid #000;
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.newspaper-table tbody tr:last-child td {
|
||||
border-bottom: 3px double #000;
|
||||
}
|
||||
|
||||
/* Clickable rows */
|
||||
.release-row {
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s ease;
|
||||
}
|
||||
|
||||
.release-row:hover {
|
||||
background: #f8f8f8;
|
||||
}
|
||||
|
||||
.release-row.expanded {
|
||||
background: #f0f0f0;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* Version cell styling */
|
||||
.version-cell {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.version-name {
|
||||
font-size: 0.85rem;
|
||||
color: #333;
|
||||
font-weight: normal;
|
||||
}
|
||||
|
||||
/* Status badges */
|
||||
.status {
|
||||
display: inline-block;
|
||||
padding: 0.2rem 0.5rem;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
}
|
||||
|
||||
.status.released {
|
||||
background: #000;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.status.development {
|
||||
background: #666;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.status.planned {
|
||||
background: #fff;
|
||||
color: #000;
|
||||
border: 1px solid #000;
|
||||
}
|
||||
|
||||
/* Details rows */
|
||||
.release-details {
|
||||
display: none;
|
||||
background: #fafafa;
|
||||
}
|
||||
|
||||
.release-details.expanded {
|
||||
display: table-row;
|
||||
}
|
||||
|
||||
.release-details td {
|
||||
padding: 0;
|
||||
border-bottom: 1px solid #000;
|
||||
}
|
||||
|
||||
.details-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
gap: 0;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
.detail-section {
|
||||
padding: 1rem;
|
||||
border-right: 1px solid #ccc;
|
||||
}
|
||||
|
||||
.detail-section:last-child {
|
||||
border-right: none;
|
||||
}
|
||||
|
||||
.detail-section h5 {
|
||||
font-size: 0.85rem;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
margin: 0 0 0.75rem 0;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.detail-section ul {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.detail-section li {
|
||||
margin-bottom: 0.5rem;
|
||||
padding-left: 1rem;
|
||||
position: relative;
|
||||
font-size: 0.85rem;
|
||||
line-height: 1.4;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.detail-section li::before {
|
||||
content: '•';
|
||||
position: absolute;
|
||||
left: 0;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
/* Responsive */
|
||||
@media (max-width: 768px) {
|
||||
.newspaper-table {
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.newspaper-table th,
|
||||
.newspaper-table td {
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.version-cell {
|
||||
flex-direction: column;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.details-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.detail-section {
|
||||
border-right: none;
|
||||
border-bottom: 1px solid #ccc;
|
||||
}
|
||||
|
||||
.detail-section:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
}
|
||||
|
||||
/* Print styles */
|
||||
@media print {
|
||||
.release-row {
|
||||
page-break-inside: avoid;
|
||||
}
|
||||
|
||||
.release-details {
|
||||
display: table-row !important;
|
||||
}
|
||||
}
|
||||
|
||||
/* Dark mode styles */
|
||||
html[data-theme="dark"] .newspaper-table {
|
||||
background: #1a1a1a;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .newspaper-table caption {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .newspaper-table th {
|
||||
background: #2a2a2a;
|
||||
border-color: #666;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .newspaper-table td {
|
||||
border-color: #666;
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .newspaper-table tbody tr:last-child td {
|
||||
border-bottom-color: #666;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .release-row:hover {
|
||||
background: #2a2a2a;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .release-row.expanded {
|
||||
background: #333;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .version-name {
|
||||
color: #999;
|
||||
}
|
||||
|
||||
/* Dark mode status badges */
|
||||
html[data-theme="dark"] .status.released {
|
||||
background: #fff;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .status.development {
|
||||
background: #999;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .status.planned {
|
||||
background: transparent;
|
||||
color: #fff;
|
||||
border-color: #fff;
|
||||
}
|
||||
|
||||
/* Dark mode details */
|
||||
html[data-theme="dark"] .release-details {
|
||||
background: #252525;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .release-details td {
|
||||
border-color: #666;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .detail-section {
|
||||
border-right-color: #444;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .detail-section h5 {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .detail-section li {
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .detail-section li::before {
|
||||
color: #999;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
html[data-theme="dark"] .detail-section {
|
||||
border-bottom-color: #444;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const releaseRows = document.querySelectorAll('.release-row');
|
||||
|
||||
releaseRows.forEach(row => {
|
||||
row.addEventListener('click', () => {
|
||||
const version = row.getAttribute('data-version');
|
||||
const detailsRow = document.querySelector(`.release-details[data-for="${version}"]`);
|
||||
|
||||
if (detailsRow) {
|
||||
// Toggle expanded state
|
||||
const isExpanded = detailsRow.classList.contains('expanded');
|
||||
|
||||
// Close all other rows
|
||||
document.querySelectorAll('.release-details').forEach(details => {
|
||||
details.classList.remove('expanded');
|
||||
});
|
||||
document.querySelectorAll('.release-row').forEach(r => {
|
||||
r.classList.remove('expanded');
|
||||
});
|
||||
|
||||
// Toggle this row
|
||||
if (!isExpanded) {
|
||||
detailsRow.classList.add('expanded');
|
||||
row.classList.add('expanded');
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
</script>
|
||||
@ -1,200 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
currentPage?: string;
|
||||
}
|
||||
|
||||
const { currentPage = 'products' } = Astro.props;
|
||||
---
|
||||
|
||||
<nav class="floating-nav">
|
||||
<div class="nav-container">
|
||||
<a href="/" class="nav-logo">👋 Jan</a>
|
||||
<div class="nav-links">
|
||||
<a href="/" class={currentPage === 'home' ? 'active' : ''}>Home</a>
|
||||
<a href="/products" class={currentPage === 'docs' ? 'active' : ''}>Docs</a>
|
||||
<a href="/prods" class={currentPage === 'products' ? 'active' : ''}>Products</a>
|
||||
</div>
|
||||
<button class="dark-mode-toggle" aria-label="Toggle dark mode">
|
||||
<span class="toggle-icon sun">☀</span>
|
||||
<span class="toggle-icon moon">☾</span>
|
||||
</button>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<style>
|
||||
.floating-nav {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
background: #fff;
|
||||
border-bottom: 1px solid #ddd;
|
||||
z-index: 1000;
|
||||
padding: 1rem 0;
|
||||
}
|
||||
|
||||
.nav-container {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
padding: 0 2rem;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.nav-logo {
|
||||
font-size: 1.25rem;
|
||||
font-weight: 700;
|
||||
color: #000;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.nav-logo:hover {
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.nav-links {
|
||||
display: flex;
|
||||
gap: 2rem;
|
||||
}
|
||||
|
||||
.nav-links a {
|
||||
color: #000;
|
||||
text-decoration: none;
|
||||
font-weight: 500;
|
||||
transition: color 0.2s ease;
|
||||
}
|
||||
|
||||
.nav-links a:hover {
|
||||
color: #0066cc;
|
||||
}
|
||||
|
||||
.nav-links a.active {
|
||||
color: #0066cc;
|
||||
border-bottom: 2px solid #0066cc;
|
||||
padding-bottom: 2px;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.nav-container {
|
||||
padding: 0 1rem;
|
||||
}
|
||||
|
||||
.nav-links {
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.nav-links a {
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
}
|
||||
|
||||
/* Dark mode toggle button */
|
||||
.dark-mode-toggle {
|
||||
background: none;
|
||||
border: 1px solid #000;
|
||||
padding: 0.5rem;
|
||||
cursor: pointer;
|
||||
font-size: 1.2rem;
|
||||
line-height: 1;
|
||||
border-radius: 4px;
|
||||
transition: all 0.2s ease;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
}
|
||||
|
||||
.dark-mode-toggle:hover {
|
||||
background: #f0f0f0;
|
||||
}
|
||||
|
||||
.toggle-icon {
|
||||
position: absolute;
|
||||
transition: opacity 0.3s ease, transform 0.3s ease;
|
||||
}
|
||||
|
||||
.toggle-icon.sun {
|
||||
opacity: 1;
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
|
||||
.toggle-icon.moon {
|
||||
opacity: 0;
|
||||
transform: rotate(-180deg);
|
||||
}
|
||||
|
||||
/* Dark mode active state */
|
||||
html[data-theme="dark"] .toggle-icon.sun {
|
||||
opacity: 0;
|
||||
transform: rotate(180deg);
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toggle-icon.moon {
|
||||
opacity: 1;
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .dark-mode-toggle {
|
||||
border-color: #fff;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .dark-mode-toggle:hover {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
/* Dark mode nav styles */
|
||||
html[data-theme="dark"] .floating-nav {
|
||||
background: #1a1a1a;
|
||||
border-bottom-color: #333;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .nav-logo {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .nav-logo:hover {
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .nav-links a {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .nav-links a:hover {
|
||||
color: #4a9eff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .nav-links a.active {
|
||||
color: #4a9eff;
|
||||
border-bottom-color: #4a9eff;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.dark-mode-toggle {
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
font-size: 1rem;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
// Dark mode toggle functionality
|
||||
const toggleButton = document.querySelector('.dark-mode-toggle');
|
||||
const html = document.documentElement;
|
||||
|
||||
// Check for saved theme preference or default to light
|
||||
const currentTheme = localStorage.getItem('theme') || 'light';
|
||||
html.setAttribute('data-theme', currentTheme);
|
||||
|
||||
toggleButton?.addEventListener('click', () => {
|
||||
const currentTheme = html.getAttribute('data-theme');
|
||||
const newTheme = currentTheme === 'dark' ? 'light' : 'dark';
|
||||
|
||||
html.setAttribute('data-theme', newTheme);
|
||||
localStorage.setItem('theme', newTheme);
|
||||
});
|
||||
</script>
|
||||
@ -1,338 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
sections?: Array<{
|
||||
id: string;
|
||||
title: string;
|
||||
level?: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
const {
|
||||
sections = [
|
||||
{ id: 'what-were-building', title: 'What We\'re Building' },
|
||||
{ id: 'two-modes-one-experience', title: 'Two Modes, One Experience' },
|
||||
{ id: 'our-product-principles', title: 'Our Product Principles' },
|
||||
{ id: 'available-on-every-device', title: 'Available on Every Device' },
|
||||
{ id: 'jan-desktop', title: 'Jan Desktop', level: 2 },
|
||||
{ id: 'jan-web', title: 'Jan Web', level: 2 },
|
||||
{ id: 'jan-mobile', title: 'Jan Mobile', level: 2 },
|
||||
{ id: 'jan-server', title: 'Jan Server', level: 2 },
|
||||
{ id: 'jan-mobile-three-modes-one-experience', title: 'Jan Mobile: Three Modes' },
|
||||
{ id: 'what-makes-jan-different', title: 'What Makes Jan Different' },
|
||||
{ id: 'development-timeline', title: 'Development Timeline' }
|
||||
]
|
||||
} = Astro.props;
|
||||
---
|
||||
|
||||
<nav class="toc" aria-label="Table of Contents">
|
||||
<div class="toc-header">
|
||||
<h3>Contents</h3>
|
||||
</div>
|
||||
<ul>
|
||||
{sections.map((section, index) => {
|
||||
// Calculate section numbering
|
||||
let sectionNumber = '';
|
||||
if (!section.level) {
|
||||
const mainSectionIndex = sections.slice(0, index + 1).filter(s => !s.level).length;
|
||||
sectionNumber = `${mainSectionIndex}.`;
|
||||
} else if (section.level === 2) {
|
||||
const parentIndex = sections.slice(0, index).reverse().findIndex(s => !s.level);
|
||||
const mainSectionIndex = sections.slice(0, index - parentIndex).filter(s => !s.level).length;
|
||||
const subSectionIndex = sections.slice(index - parentIndex, index + 1).filter(s => s.level === 2).length;
|
||||
sectionNumber = `${mainSectionIndex}.${subSectionIndex}`;
|
||||
}
|
||||
|
||||
return (
|
||||
<li class={section.level ? `level-${section.level}` : ''}>
|
||||
<span class="section-number">{sectionNumber}</span>
|
||||
<a href={`#${section.id}`} data-section={section.id}>
|
||||
{section.title}
|
||||
</a>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<style>
|
||||
.toc {
|
||||
position: fixed;
|
||||
right: 2rem;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
width: 280px;
|
||||
max-height: 70vh;
|
||||
overflow-y: auto;
|
||||
background: #fafafa;
|
||||
border: 3px double #000;
|
||||
padding: 0;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
|
||||
font-family: Georgia, 'Times New Roman', serif;
|
||||
}
|
||||
|
||||
.toc-header {
|
||||
background: #f0f0f0;
|
||||
border-bottom: 1px solid #000;
|
||||
padding: 1rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.toc h3 {
|
||||
font-size: 1.1rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.1em;
|
||||
margin: 0;
|
||||
color: #000;
|
||||
font-weight: 900;
|
||||
font-family: Georgia, 'Times New Roman', serif;
|
||||
}
|
||||
|
||||
.toc ul {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.toc li {
|
||||
margin-bottom: 0;
|
||||
border-bottom: 1px dotted #999;
|
||||
padding: 0.5rem 0;
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.toc li:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.toc li.level-2 {
|
||||
padding-left: 1.5rem;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.toc li.level-2 .section-number {
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.toc li.level-3 {
|
||||
padding-left: 3rem;
|
||||
}
|
||||
|
||||
.section-number {
|
||||
font-weight: 700;
|
||||
font-size: 0.9rem;
|
||||
color: #000;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.toc a {
|
||||
color: #000;
|
||||
text-decoration: none;
|
||||
font-size: 0.95rem;
|
||||
display: block;
|
||||
transition: all 0.2s ease;
|
||||
line-height: 1.3;
|
||||
font-family: Georgia, 'Times New Roman', serif;
|
||||
}
|
||||
|
||||
.toc a:hover {
|
||||
color: #000;
|
||||
text-decoration: underline;
|
||||
text-decoration-style: solid;
|
||||
text-underline-offset: 2px;
|
||||
}
|
||||
|
||||
.toc a.active {
|
||||
color: #000;
|
||||
font-weight: 700;
|
||||
text-decoration: underline;
|
||||
text-decoration-thickness: 2px;
|
||||
}
|
||||
|
||||
/* Scrollbar styling - classic newspaper style */
|
||||
.toc::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
}
|
||||
|
||||
.toc::-webkit-scrollbar-track {
|
||||
background: #f0f0f0;
|
||||
border-left: 1px solid #ccc;
|
||||
}
|
||||
|
||||
.toc::-webkit-scrollbar-thumb {
|
||||
background: #999;
|
||||
border-left: 1px solid #666;
|
||||
}
|
||||
|
||||
.toc::-webkit-scrollbar-thumb:hover {
|
||||
background: #666;
|
||||
}
|
||||
|
||||
/* Add newspaper-style ornament */
|
||||
.toc::before {
|
||||
content: '§';
|
||||
position: absolute;
|
||||
top: 0.5rem;
|
||||
right: 1rem;
|
||||
font-size: 1.5rem;
|
||||
color: #666;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.toc::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
bottom: 0;
|
||||
left: 1rem;
|
||||
right: 1rem;
|
||||
height: 3px;
|
||||
background: repeating-linear-gradient(
|
||||
to right,
|
||||
#000,
|
||||
#000 2px,
|
||||
transparent 2px,
|
||||
transparent 4px
|
||||
);
|
||||
}
|
||||
|
||||
/* Dark mode styles */
|
||||
html[data-theme="dark"] .toc {
|
||||
background: #2a2a2a;
|
||||
border-color: #666;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc-header {
|
||||
background: #333;
|
||||
border-bottom-color: #666;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc h3 {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc li {
|
||||
border-bottom-color: #444;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .section-number {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc a {
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc a:hover {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc a.active {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc::before {
|
||||
color: #999;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc::after {
|
||||
background: repeating-linear-gradient(
|
||||
to right,
|
||||
#666,
|
||||
#666 2px,
|
||||
transparent 2px,
|
||||
transparent 4px
|
||||
);
|
||||
}
|
||||
|
||||
/* Dark mode scrollbar */
|
||||
html[data-theme="dark"] .toc::-webkit-scrollbar-track {
|
||||
background: #333;
|
||||
border-left-color: #444;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc::-webkit-scrollbar-thumb {
|
||||
background: #666;
|
||||
border-left-color: #888;
|
||||
}
|
||||
|
||||
html[data-theme="dark"] .toc::-webkit-scrollbar-thumb:hover {
|
||||
background: #888;
|
||||
}
|
||||
|
||||
/* Hide on smaller screens */
|
||||
@media (max-width: 1280px) {
|
||||
.toc {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const tocLinks = document.querySelectorAll('.toc a');
|
||||
const sections = document.querySelectorAll('h2[id], h3[id]');
|
||||
|
||||
// Smooth scrolling
|
||||
tocLinks.forEach(link => {
|
||||
link.addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
const targetId = link.getAttribute('href')?.substring(1);
|
||||
const targetElement = document.getElementById(targetId);
|
||||
|
||||
if (targetElement) {
|
||||
const offset = 80; // Account for fixed header
|
||||
const elementPosition = targetElement.getBoundingClientRect().top + window.pageYOffset;
|
||||
const offsetPosition = elementPosition - offset;
|
||||
|
||||
window.scrollTo({
|
||||
top: offsetPosition,
|
||||
behavior: 'smooth'
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Update active section on scroll
|
||||
const updateActiveSection = () => {
|
||||
const scrollTop = window.pageYOffset + 100;
|
||||
|
||||
let activeSection = null;
|
||||
|
||||
// Find the current active section
|
||||
sections.forEach(section => {
|
||||
const rect = section.getBoundingClientRect();
|
||||
const absoluteTop = rect.top + window.pageYOffset;
|
||||
|
||||
if (absoluteTop <= scrollTop) {
|
||||
activeSection = section;
|
||||
}
|
||||
});
|
||||
|
||||
tocLinks.forEach(link => {
|
||||
link.classList.remove('active');
|
||||
if (activeSection && link.getAttribute('href') === `#${activeSection.id}`) {
|
||||
link.classList.add('active');
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Throttled scroll handler
|
||||
let ticking = false;
|
||||
const handleScroll = () => {
|
||||
if (!ticking) {
|
||||
requestAnimationFrame(() => {
|
||||
updateActiveSection();
|
||||
ticking = false;
|
||||
});
|
||||
ticking = true;
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('scroll', handleScroll);
|
||||
|
||||
// Initial update
|
||||
updateActiveSection();
|
||||
});
|
||||
</script>
|
||||
@ -1,147 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
status: 'active' | 'warning' | 'success' | 'error' | 'idle';
|
||||
label: string;
|
||||
pulse?: boolean;
|
||||
size?: 'small' | 'medium' | 'large';
|
||||
}
|
||||
|
||||
const { status, label, pulse = true, size = 'medium' } = Astro.props;
|
||||
|
||||
const statusColors = {
|
||||
active: '#00ff41',
|
||||
warning: '#ffb000',
|
||||
success: '#00ff41',
|
||||
error: '#ff0040',
|
||||
idle: '#888888'
|
||||
};
|
||||
|
||||
const statusColor = statusColors[status];
|
||||
---
|
||||
|
||||
<div class={`status-indicator ${size} ${pulse ? 'pulse' : ''}`}>
|
||||
<div class="status-dot" style={`background-color: ${statusColor}`}></div>
|
||||
<span class="status-label">{label}</span>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.status-indicator {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
}
|
||||
|
||||
.status-dot {
|
||||
border-radius: 50%;
|
||||
position: relative;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.status-indicator.small .status-dot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
}
|
||||
|
||||
.status-indicator.medium .status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
}
|
||||
|
||||
.status-indicator.large .status-dot {
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
}
|
||||
|
||||
.status-label {
|
||||
color: #cccccc;
|
||||
font-size: 0.9rem;
|
||||
font-weight: 400;
|
||||
letter-spacing: 0.02em;
|
||||
}
|
||||
|
||||
.status-indicator.small .status-label {
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.status-indicator.large .status-label {
|
||||
font-size: 1rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Pulse animation */
|
||||
.status-indicator.pulse .status-dot {
|
||||
animation: statusPulse 2s infinite;
|
||||
}
|
||||
|
||||
.status-indicator.pulse .status-dot::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: -2px;
|
||||
left: -2px;
|
||||
right: -2px;
|
||||
bottom: -2px;
|
||||
border-radius: 50%;
|
||||
background: inherit;
|
||||
opacity: 0.3;
|
||||
animation: statusRipple 2s infinite;
|
||||
}
|
||||
|
||||
@keyframes statusPulse {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
transform: scale(1);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.7;
|
||||
transform: scale(1.1);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes statusRipple {
|
||||
0% {
|
||||
transform: scale(1);
|
||||
opacity: 0.3;
|
||||
}
|
||||
100% {
|
||||
transform: scale(2);
|
||||
opacity: 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Hover effects */
|
||||
.status-indicator:hover .status-dot {
|
||||
transform: scale(1.2);
|
||||
filter: brightness(1.2);
|
||||
}
|
||||
|
||||
.status-indicator:hover .status-label {
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
/* Terminal-style glow effect */
|
||||
.status-dot {
|
||||
box-shadow:
|
||||
0 0 5px currentColor,
|
||||
inset 0 0 5px rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
/* Accessibility */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.status-dot,
|
||||
.status-dot::before {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
|
||||
/* Dark mode adjustments */
|
||||
@media (prefers-color-scheme: light) {
|
||||
.status-label {
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
.status-indicator:hover .status-label {
|
||||
color: #000000;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@ -1,112 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
class?: string;
|
||||
}
|
||||
|
||||
const { class: className } = Astro.props;
|
||||
---
|
||||
|
||||
<div class={`steps-container ${className || ''}`}>
|
||||
<slot />
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.steps-container {
|
||||
margin: 2rem 0;
|
||||
padding: 1.5rem;
|
||||
background-color: var(--sl-color-gray-6);
|
||||
border-radius: 0.5rem;
|
||||
border: 1px solid var(--sl-color-gray-5);
|
||||
}
|
||||
|
||||
/* Light mode adjustments */
|
||||
:global([data-theme="light"]) .steps-container {
|
||||
background-color: var(--sl-color-gray-7);
|
||||
border-color: var(--sl-color-gray-6);
|
||||
}
|
||||
|
||||
/* Style the headers inside steps */
|
||||
.steps-container :global(h2),
|
||||
.steps-container :global(h3) {
|
||||
margin-top: 1.5rem;
|
||||
margin-bottom: 0.75rem;
|
||||
color: var(--sl-color-text-accent);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.steps-container :global(h2:first-child),
|
||||
.steps-container :global(h3:first-child) {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
/* Style lists inside steps */
|
||||
.steps-container :global(ul) {
|
||||
margin: 0.5rem 0;
|
||||
padding-left: 1.5rem;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.steps-container :global(ul li) {
|
||||
position: relative;
|
||||
margin: 0.5rem 0;
|
||||
padding-left: 1.25rem;
|
||||
}
|
||||
|
||||
/* Custom bullet points */
|
||||
.steps-container :global(ul li::before) {
|
||||
content: '→';
|
||||
position: absolute;
|
||||
left: 0;
|
||||
color: var(--sl-color-text-accent);
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
/* Style links inside steps */
|
||||
.steps-container :global(a) {
|
||||
color: var(--sl-color-text);
|
||||
text-decoration: none;
|
||||
transition: color 0.2s ease;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.steps-container :global(a:hover) {
|
||||
color: var(--sl-color-text-accent);
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
/* Add spacing between step groups */
|
||||
.steps-container :global(h2:not(:first-child)),
|
||||
.steps-container :global(h3:not(:first-child)) {
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid var(--sl-color-hairline);
|
||||
}
|
||||
|
||||
/* Numbered steps styling */
|
||||
.steps-container :global(h2) {
|
||||
counter-increment: step-counter;
|
||||
position: relative;
|
||||
padding-left: 2.5rem;
|
||||
}
|
||||
|
||||
.steps-container {
|
||||
counter-reset: step-counter;
|
||||
}
|
||||
|
||||
.steps-container :global(h2::before) {
|
||||
content: counter(step-counter);
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
width: 2rem;
|
||||
height: 2rem;
|
||||
background-color: var(--sl-color-accent);
|
||||
color: var(--sl-color-white);
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 0.875rem;
|
||||
font-weight: bold;
|
||||
}
|
||||
</style>
|
||||
@ -1,60 +0,0 @@
|
||||
---
|
||||
export interface Props {
|
||||
id: string;
|
||||
title?: string;
|
||||
class?: string;
|
||||
}
|
||||
|
||||
const { id, title = 'YouTube video player', class: className } = Astro.props;
|
||||
|
||||
// Extract video ID and handle both formats:
|
||||
// - Simple ID: "4mvHgLy_YV8"
|
||||
// - ID with params: "4mvHgLy_YV8?si=74cmdMmcH3gmpv0R"
|
||||
const videoId = id.split('?')[0];
|
||||
const params = id.includes('?') ? '?' + id.split('?')[1] : '';
|
||||
---
|
||||
|
||||
<div class={`youtube-container ${className || ''}`}>
|
||||
<iframe
|
||||
src={`https://www.youtube.com/embed/${videoId}${params}`}
|
||||
title={title}
|
||||
frameborder="0"
|
||||
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
|
||||
allowfullscreen
|
||||
loading="lazy"
|
||||
></iframe>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.youtube-container {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
padding-bottom: 56.25%; /* 16:9 aspect ratio */
|
||||
height: 0;
|
||||
overflow: hidden;
|
||||
margin: 1.5rem 0;
|
||||
border-radius: 0.5rem;
|
||||
background-color: var(--sl-color-gray-6);
|
||||
}
|
||||
|
||||
.youtube-container iframe {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border: 0;
|
||||
border-radius: 0.5rem;
|
||||
}
|
||||
|
||||
/* Light mode adjustments */
|
||||
:global([data-theme="light"]) .youtube-container {
|
||||
background-color: var(--sl-color-gray-7);
|
||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
/* Ensure proper spacing in content */
|
||||
:global(.sl-markdown-content) .youtube-container {
|
||||
margin: 2rem 0;
|
||||
}
|
||||
</style>
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 885 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 3.9 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 742 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 163 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.0 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 415 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 725 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 130 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 795 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.6 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 162 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 218 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 90 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 37 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 141 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 240 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 586 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 55 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 899 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 117 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 453 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 231 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 143 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 138 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 120 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 673 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 374 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 192 KiB |
@ -1,321 +0,0 @@
|
||||
---
|
||||
title: Benchmarking NVIDIA TensorRT-LLM
|
||||
description: This post compares the performance of TensorRT-LLM and llama.cpp on consumer NVIDIA GPUs, highlighting the trade-offs among speed, resource usage, and convenience.
|
||||
tags: Nvidia, TensorRT-LLM, llama.cpp, rtx3090, rtx4090, "inference engine"
|
||||
categories: research
|
||||
ogImage: assets/images/general/og-throughput-benchmark.png
|
||||
date: 2024-04-29
|
||||
---
|
||||
|
||||
import { Aside } from '@astrojs/starlight/components'
|
||||
|
||||
|
||||
import throughputComparison from '@/assets/blog/throughput_Comparison.png';
|
||||
import img4090s from '@/assets/blog/4090s.png';
|
||||
import og4090s from '@/assets/blog/og-4090s.webp';
|
||||
import img3090s from '@/assets/blog/3090s.jpg';
|
||||
import img4070s from '@/assets/blog/4070s.jpg';
|
||||
import egpu from '@/assets/blog/egpu.jpg';
|
||||
|
||||
|
||||
Jan now supports [NVIDIA TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) in addition to [llama.cpp](https://github.com/ggerganov/llama.cpp), making Jan multi-engine and ultra-fast for users with Nvidia GPUs.
|
||||
|
||||
We've been excited for TensorRT-LLM for a while, and [had a lot of fun implementing it](https://github.com/menloresearch/nitro-tensorrt-llm). As part of the process, we've run some benchmarks, to see how TensorRT-LLM fares on consumer hardware (e.g. [4090s](https://www.nvidia.com/en-us/geforce/graphics-cards/40-series/), [3090s](https://www.nvidia.com/en-us/geforce/graphics-cards/30-series/)) we commonly see in the [Jan's hardware community](https://discord.com/channels/1107178041848909847/1201834752206974996).
|
||||
|
||||
<Aside type="note" >
|
||||
**Give it a try!** Jan's [TensorRT-LLM extension](/docs/built-in/tensorrt-llm) is available in Jan v0.4.9 and up ([see more](/docs/built-in/tensorrt-llm)). We precompiled some TensorRT-LLM models for you to try: `Mistral 7b`, `TinyLlama-1.1b`, `TinyJensen-1.1b` 😂
|
||||
|
||||
Bugs or feedback? Let us know on [GitHub](https://github.com/menloresearch/jan) or via [Discord](https://discord.com/channels/1107178041848909847/1201832734704795688).
|
||||
</Aside>
|
||||
|
||||
<Aside type="note" >
|
||||
**An interesting aside:** Jan actually started out in June 2023 building on [NVIDIA FastTransformer](https://github.com/NVIDIA/FasterTransformer), the precursor library to TensorRT-LLM. TensorRT-LLM was released in September 2023, making it a very young library. We’re excited to see its roadmap develop!
|
||||
</Aside>
|
||||
|
||||
## Key Findings
|
||||
|
||||
<img src={throughputComparison.src} alt="image" />
|
||||
|
||||
TensorRT-LLM was:
|
||||
|
||||
- **30-70% faster** than llama.cpp on the same hardware
|
||||
- **Consumes less memory on consecutive runs** and **marginally more GPU VRAM utilization** than llama.cpp
|
||||
- **20%+ smaller compiled model sizes** than llama.cpp
|
||||
- **Less convenient** as models have to be compiled for a specific OS and GPU architecture, vs. llama.cpp's "Compile once, run everywhere" portability
|
||||
- **Less accessible** as it does not support older-generation NVIDIA GPUs
|
||||
|
||||
## Why TensorRT-LLM?
|
||||
|
||||
[TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) is Nvidia's open-source inference library that incorporates Nvidia's proprietary optimizations beyond the open-source [cuBLAS](https://developer.nvidia.com/cublas) library.
|
||||
|
||||
As compared to [llama.cpp](https://github.com/ggerganov/llama.cpp), which today dominates Desktop AI as a cross-platform inference engine, TensorRT-LLM is highly optimized for Nvidia GPUs. While llama.cpp compiles models into a [single, generalizable CUDA "backend"](https://github.com/ggerganov/llama.cpp/blob/master/ggml-cuda.cu) that can run on a wide range of Nvidia GPUs, TensorRT-LLM compiles models into a [GPU-specific execution graph](https://www.baseten.co/blog/high-performance-ml-inference-with-nvidia-tensorrt/) that is highly optimized for that specific GPU's Tensor Cores, CUDA cores, VRAM and memory bandwidth.
|
||||
|
||||
TensorRT-LLM is typically used in datacenter-grade GPUs, where it produces a [face-melting 10,000 tokens/s](https://nvidia.github.io/TensorRT-LLM/blogs/H100vsA100.html) on [NVIDIA H100 Tensor Core GPUs](https://www.nvidia.com/en-us/data-center/h100/). We were curious for how TensorRT-LLM performs on consumer-grade GPUs, and gave it a spin.
|
||||
|
||||
| Llama.cpp | TensorRT-LLM |
|
||||
| ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Baseline | Blazing fast (30-70% faster) |
|
||||
| Compile once, run cross-platform | Compiled and highly optimized for specific GPU architecture |
|
||||
| Generalizable and Portable | Highly-optimized |
|
||||
| Model compiles to [single, generalizable CUDA "backend"](https://github.com/ggerganov/llama.cpp/blob/master/ggml-cuda.cu) | Model compiles to [GPU-specific execution graph](https://www.baseten.co/blog/high-performance-ml-inference-with-nvidia-tensorrt/) |
|
||||
|
||||
## Experiment Setup
|
||||
|
||||
We ran the experiment using standardized inference requests in a sandboxed environment:
|
||||
- **Model**: Mistral 7b model, compiled and quantized at a comparable `int4` quantization.
|
||||
- **Test runs**: 5 batches of 10 runs each, per inference engine, on a bare metal PC with no other applications.
|
||||
- **Parameters**: User defaults, i.e. `batch_size 1`, `input_len 2048` and `output_len 512`
|
||||
- **Measurements**:
|
||||
- CPU, memory from Jan system monitor
|
||||
- GPU VRAM utilization metrics from `nvidia-smi`, and taken over an interval of 14 seconds.
|
||||
- Throughput (token/sec) using [Jan's built-in Tokens/sec perf stat](https://github.com/search?q=repo%3Ajanhq%2Fjan%20timeDiffInSeconds&type=code).
|
||||
|
||||
<Aside type="tip">
|
||||
We picked a [batch size of 1](https://www.reddit.com/r/LocalLLaMA/comments/17sbwo5/what_does_batch_size_mean_in_inference/) to simulate realistic real-world use cases, as Jan users are likely to make one inference request at a time. We also used a `input_len` of 2048 and `output_len` of 512 to simulate a typical chatbot use case.
|
||||
</Aside>
|
||||
|
||||
<Aside type="tip">
|
||||
**Further Research**: We found GPU VRAM Utilization to be difficult to measure, with `nvidia-smi` on Windows producing a different result from Windows Task Manager's Resource Monitor for NVIDIA GPUs. After some [research](https://forums.developer.nvidia.com/t/how-to-evaluate-gpu-utilization-usage-on-windows/245451/3), we have gone with `nvidia-smi`'s measurement.
|
||||
</Aside>
|
||||
|
||||
### Hardware Selection
|
||||
|
||||
We chose the following GPUs based on our users' preferences:
|
||||
|
||||
| NVIDIA GPU | VRAM (GB) | CUDA Cores | Tensor Cores | Memory Bus Width (bit) | Memory Bandwidth (GB/s) | Connection (GB/s) |
|
||||
| --------------------------------- | --------- | ---------- | ------------ | ---------------------- | ----------------------- | -------------------------------------------- |
|
||||
| GeForce RTX 4090 (Ada) | 24 | 16,384 | 512 | 384 | ~1000 | PCIe4.0 x16 (~32) |
|
||||
| GeForce RTX 3090 (Ampere) | 24 | 10,496 | 328 | 384 | 935.8 | PCIe4.0 x16 (~32) |
|
||||
| GeForce RTX 4070 Laptop GPU (Ada) | 8 | 7680 | 144 | 192 | 272 | PCIe4.0 x4 (~8) |
|
||||
| GeForce RTX 4090 eGPU (Ada) | 24 | 16,384 | 512 | 384 | ~1000 | Thunderbolt 3 connected to a USB4 USB-C port ([~1.25-5?](https://www.cablematters.com/Blog/Thunderbolt/usb4-vs-thunderbolt-3)) |
|
||||
|
||||
### llama.cpp Setup
|
||||
|
||||
- llama.cpp commit [15499eb](https://github.com/ggerganov/llama.cpp/commit/15499eb94227401bdc8875da6eb85c15d37068f7)
|
||||
- We used `Mistral-7b-q4_k_m` in `GGUF` with `ngl` at `100`
|
||||
|
||||
<Aside type="tip">
|
||||
Note: `ngl` is the abbreviation of `Number of GPU Layers` with the range from `0` as no GPU acceleration to `120` as full on GPU for some big models.
|
||||
</Aside>
|
||||
|
||||
### TensorRT-LLM Setup
|
||||
|
||||
- TensorRT-LLM version [0.7.1](https://github.com/NVIDIA/TensorRT-LLM/releases/tag/v0.7.1) and build on Windows
|
||||
- For TensorRT-LLM, we used `Mistral-7b-int4 AWQ`
|
||||
- We ran TensorRT-LLM with `free_gpu_memory_fraction` to test it with the lowest VRAM consumption
|
||||
- Note: We picked AWQ for TensorRT-LLM to be a closer comparison to GGUF's Q4.
|
||||
|
||||
## Results
|
||||
|
||||
### NVIDIA GeForce RTX 4090 GPU
|
||||
|
||||
<img src={img4090s.src} alt="image" />
|
||||
*Jan is built on this Dual-4090 workstation, which recently got upgraded to a nice case*
|
||||
|
||||
<img src={og4090s.src} alt="image" />
|
||||
*The original case (or lack thereof) for our Dual-4090 cluster, as posted on [r/localllama](https://www.reddit.com/r/LocalLLaMA/comments/16lxt6a/case_for_dual_4090s/)*
|
||||
|
||||
<Aside type="note" >
|
||||
**Hardware Details**
|
||||
- CPU: Intel 13th series
|
||||
- GPU: NVIDIA GeForce RTX 4090 (Ada - sm 89)
|
||||
- RAM: 32GB
|
||||
- OS: Windows 11 Pro
|
||||
|
||||
**Model Details**
|
||||
- llama.cpp model: Mistral 7B v0.2 GGUF Q4_K_M
|
||||
- TensorRT-LLM model: Mistral 7B v0.2 AWQ, quantized for single GPU (Ada)
|
||||
</Aside>
|
||||
|
||||
For this test, we used Jan's [Dual-4090 workstation](https://www.reddit.com/r/LocalLLaMA/comments/16lxt6a/case_for_dual_4090s/), which our engineers timeshare to build Jan.
|
||||
|
||||
The [NVIDIA GeForce RTX 4090](https://www.nvidia.com/en-us/geforce/graphics-cards/40-series/) is the latest top-of-the-line desktop GPU, with an MSRP of $1,599, and uses the Ada architecture. It has a ~1000 GB/s memory bandwidth within VRAM, and a PCIe4 x16 lane (~32 GB/s) between the GPU and the CPU.
|
||||
|
||||
| Metrics | GGUF (using CPU) | GGUF (using GPU) | TensorRT-LLM | How TensorRT-LLM Compares |
|
||||
| ------------------------ | ---------------- | ---------------- | ------------ | ------------------------- |
|
||||
| Throughput (token/s) | 14.0 | 100.43 | 170.63 | ✅ 69.89% faster |
|
||||
| Max GPU Utilization (%) | N/A | 83.50 | 88.50 | 5.99% more |
|
||||
| Max VRAM Utilization (%) | N/A | 64 | 72.1 | 12.66% more |
|
||||
| Avg RAM Used (GB) | 0.611 | 7.105 | 4.98 | ✅ 29.88% less |
|
||||
| Disk Size (GB) | 4.07 | 4.06 | 3.05 | ✅ 24.88% smaller |
|
||||
|
||||
TensorRT-LLM was almost 70% faster than llama.cpp by building the model for the GeForce RTX 4090 GPU’s Ada architecture for optimal graph execution, fully utilizing the 512 Tensor Cores, 16,384 CUDA cores, and 1,000 GB/s of memory bandwidth.
|
||||
|
||||
The intuition for why llama.cpp is slower is because it compiles a model into a [single, generalizable CUDA “backend”](https://github.com/ggerganov/llama.cpp/blob/master/ggml-cuda.cu) that can run on many NVIDIA GPUs. Doing so requires llama.cpp to sacrifice all the optimizations that TensorRT-LLM makes with its compilation to a GPU-specific execution graph.
|
||||
|
||||
### NVIDIA GeForce RTX 3090 GPU
|
||||
|
||||
<img src={img3090s.src} alt="image" />
|
||||
*Our 3090 Machine, now used by one of our engineers to build Jan*
|
||||
|
||||
<Aside type="note" >
|
||||
**Hardware Details**
|
||||
- CPU: Intel 13th series
|
||||
- GPU: NVIDIA GeForce RTX 3090 (Ampere - sm 86)
|
||||
- RAM: 64GB
|
||||
- OS: Windows 11 Pro
|
||||
|
||||
**Model Details**
|
||||
- llama.cpp model: Mistral 7B v0.2 GGUF Q4_K_M
|
||||
- TensorRT-LLM model: Mistral 7B v0.2 AWQ, quantized for single GPU (Ampere)
|
||||
</Aside>
|
||||
|
||||
The [NVIDIA's GeForce RTX 3090](https://www.nvidia.com/en-us/geforce/graphics-cards/30-series/rtx-3090-3090ti/) is a popular desktop GPU, and retails for approximately $1,500 (as of April 24). It uses the NVIDIA Ampere architecture. As compared to its successor GeForce RTX 4090, it has 33% fewer CUDA cores (10,496) and Tensor Cores (328) and 7% less memory bandwidth (~930 GB/s).
|
||||
|
||||
| Metrics | GGUF (using CPU) | GGUF (using GPU) | TensorRT-LLM | How TensorRT-LLM Compares |
|
||||
| ------------------------ | ---------------- | ---------------- | ------------ | ------------------------- |
|
||||
| Throughput (token/s) | 11.42 | 88.70 | 144.19 | ✅ 62.57% faster |
|
||||
| Max GPU Utilization (%) | N/A | 80.40 | 89.10 | 10.82% more |
|
||||
| Max VRAM Utilization (%) | N/A | 66.80 | 76.20 | 14.07% more |
|
||||
| Avg RAM Used (GB) | 0.611 | 2.60 | 0.98 | 62.41%% less |
|
||||
| Disk Size (GB) | 4.07 | 4.06 | 3.05 | ✅ 24.88% smaller |
|
||||
|
||||
Interestingly, the GeForce RTX 3090 was only 16.6% slower compared with the GeForce RTX 4090. On TPS, TensorRT-LLM outperformed llama.cpp by 62.57%. Curiously, it also used negligible RAM for subsequent inference requests after the initial model warmup.
|
||||
|
||||
### NVIDIA GeForce RTX 4070 Laptop GPU
|
||||
|
||||
<img src={img4070s.src} alt="image" />
|
||||
|
||||
<Aside type="note" >
|
||||
**Hardware Details**
|
||||
- Laptop: Razer Blade 14
|
||||
- CPU: AMD Ryzen™ 9 8945HS, 8C/16T
|
||||
- GPU: NVIDIA GeForce RTX 4070 Laptop GPU (Ada - sm 89) on PCIe 4.0 x16 (32 GB/s)
|
||||
- RAM: 32GB
|
||||
- OS: Windows 11 Pro
|
||||
|
||||
**Model Details**
|
||||
- llama.cpp model: Mistral 7B v0.2 GGUF `Q4_K_M`
|
||||
- TensorRT-LLM model: Mistral 7B v0.2 AWQ, quantized for single GPU (Ada)
|
||||
</Aside>
|
||||
|
||||
We also benchmarked an NVIDIA GeForce RTX 4070 Laptop GPU with 8gb of VRAM, which is a popular configuration among Jan users. Laptop GPUs are less powerful than their desktop counterparts, as they trade portability for reduced energy consumption and thermal constraints.
|
||||
|
||||
| Metrics | GGUF (using CPU) | GGUF (using GPU) | TensorRT-LLM | Difference on GPU |
|
||||
| ------------------------ | ---------------- | ---------------- | ------------ | ----------------- |
|
||||
| Throughput (token/s) | 11.57 | 39.70 | 51.57 | ✅ 29.9% faster |
|
||||
| Max GPU Utilization (%) | N/A | 80.00 | 84.67 | 5.83% more |
|
||||
| Max VRAM Utilization (%) | N/A | 72.78 | 81.22 | 11.60% more |
|
||||
| Avg RAM Used (GB) | 4.49 | 4.44 | 1.04 | ✅ 76.55%% less |
|
||||
| Disk Size (GB) | 4.07 | 4.06 | 3.05 | ✅ 24.88% smaller |
|
||||
|
||||
TensorRT-LLM on the laptop dGPU was 29.9% faster in tokens per second throughput than llama.cpp, but significantly slower than the desktop GPUs.
|
||||
|
||||
The intuition for this is fairly simple: the GeForce RTX 4070 Laptop GPU has 53.1% fewer CUDA cores and Tensor Cores (compared to the 4090), and less VRAM (8gb vs. 24gb). This reduces the surface area for GPU-specific optimizations for TensorRT-LLM.
|
||||
|
||||
The GeForce RTX 4070 Laptop GPU is also ~70% slower than the GeForce RTX 4090 desktop GPU, showing the hardware effect of less electricity draw, less VRAM, and thermal constraints on inference speed.
|
||||
|
||||
### Laptop with NVIDIA GeForce RTX 4090 eGPU
|
||||
|
||||
<img src={egpu.src} alt="image" />
|
||||
|
||||
<Aside type="note" >
|
||||
**Hardware Details**
|
||||
- Laptop: Razer Blade 14
|
||||
- eGPU box: AORUS RTX 4090 GAMING BOX
|
||||
- CPU: AMD Ryzen™ 9 8945HS, 8C/16T
|
||||
- GPU: NVIDIA GeForce RTX 4090 (Ada - sm 89) on eGPU with Thunderbolt 3 connection
|
||||
- RAM: 32GB
|
||||
- OS: Windows 11 Pro
|
||||
|
||||
**Model Details**
|
||||
- llama.cpp model: Mistral 7B v0.2 GGUF `Q4_K_M`
|
||||
- TensorRT-LLM model: Mistral 7B v0.2 AWQ, quantized for single GPU (Ampere)
|
||||
</Aside>
|
||||
|
||||
Our last benchmark was to experiment with an [Asus RTX 4090 eGPU](https://www.gigabyte.com/Graphics-Card/GV-N4090IXEB-24GD), that was connected via a [Thunderbolt 3 port](https://www.gigabyte.com/Graphics-Card/GV-N4090IXEB-24GD) to the [Razer Blade 14's USB4 port](https://www.razer.com/sg-en/gaming-laptops/razer-blade-14). Theoretically, the results should be fairly similar to the GeForce RTX 4090 desktop GPU as they have identical underlying GPUs, but with very different connection speeds.
|
||||
|
||||
We thought it would be an interesting to see how TensorRT-LLM handles a 68.4% reduction in communication bandwidth between the CPU and GPU:
|
||||
- Thunderbolt 3 connection (1.25-5 GB/s?) for eGPUs
|
||||
- PCIe 4.0 x4 (~8 GB/s) for "on device" desktops
|
||||
|
||||
<Aside type="tip">
|
||||
**Help Needed:** We are actually not 100% sure what the actual Connection Speed is [between Thunderbolt 3 to USB4](https://www.cablematters.com/Blog/Thunderbolt/usb4-vs-thunderbolt-3), as the actual speed seems different from the advertised speed of 5 GB/s. There seem to be a lot of factors involved, including the actual cable itself. We'd love if someone in the community can guide us!
|
||||
</Aside>
|
||||
|
||||
Overall, we used mid-to-high-end NVIDIA desktop GPUs for our tests, as TensorRT-LLM’s performance enhancements are most apparent on bigger VRAMs. For users with lower-spec machines, llama.cpp is better.
|
||||
|
||||
| Metrics | GGUF (using CPU) | GGUF (using GPU) | TensorRT-LLM | Difference on GPU |
|
||||
| ------------------------ | ---------------- | ---------------- | ------------ | ----------------- |
|
||||
| Throughput (token/s) | 11.56 | 62.22 | 104.95 | ✅ 68.66% faster |
|
||||
| Max VRAM Utilization (%) | 0 | 65 | 99 | 52.31% more |
|
||||
| RAM Used (GB) | 0.611 | 5.38 | 4.11 | ✅ 23.61% less |
|
||||
| Disk Size (GB) | 4.07 | 4.06 | 3.05 | ✅ 24.88% smaller |
|
||||
|
||||
The Thunderbolt 3 eGPU had a 38.5% lower tokens/s as compared to the PCIe4.0 x16 connected GPU. But the % speedup vs. llama.cpp was similar, at around 69%.
|
||||
|
||||
Interestingly, the VRAM used with the eGPU was variably higher. Our hypothesis is that the slower communication bandwidth results in more VRAM being allocated, as memory is released mostly slowly as well.
|
||||
|
||||
## Conclusion
|
||||
|
||||
### Token Speed
|
||||
|
||||
<img src={throughputComparison.src} alt="image" />
|
||||
|
||||
| Throughput (Higher is Better) | TensorRT-LLM | Llama.cpp | % Difference |
|
||||
| ---------------------------------- | --------------- | ----------- | ------------- |
|
||||
| GeForce RTX 4090 desktop GPU | ✅ 170.63t/s | 100.43t/s | 69.89% faster |
|
||||
| GeForce RTX 3090 desktop GPU | ✅ 144.19t/s | 88.70t/s | 62.57% faster |
|
||||
| GeForce RTX 4090 eGPU | ✅ 104.95t/s | 62.22t/s | 68.66% faster |
|
||||
| GeForce RTX 4070 Laptop GPU | ✅ 51.57t/s | 39.70t/s | 29.90% faster |
|
||||
| Laptop AMD Ryzen™ 9 8945HS, 8C/16T | (Not supported) | ✅ 11.57t/s | |
|
||||
|
||||
- TensorRT-LLM is up to **70% faster** than llama.cpp on desktop GPUs (e.g. 3090 GPU, 4090 GPUs) while using less RAM & CPU (but more fully utilizing VRAM)
|
||||
- TensorRT-LLM is up to **30% faster** on laptop GPUs (e.g. 4070 GPUs) with smaller VRAM
|
||||
|
||||
### Max VRAM Utilization
|
||||
|
||||
| Average VRAM utilization % | TensorRT-LLM | Llama.cpp | % Difference |
|
||||
| ---------------------------- | ------------ | --------- | ------------ |
|
||||
| GeForce RTX 4090 desktop GPU | 72.10 | 64.00 | 12.66% more |
|
||||
| GeForce RTX 3090 desktop GPU | 76.20 | 66.80 | 14.07% more |
|
||||
| GeForce RTX 4070 Laptop GPU | 81.22 | 72.78 | 11.06% more |
|
||||
| GeForce RTX 4090 eGPU | N/A | N/A | N/A |
|
||||
|
||||
- TensorRT-LLM used marginally more average VRAM utilization at peak utilization vs. llama.cpp (up to 14%). Though this could have interesting implications on consuming more electricity over time.
|
||||
- Note: we used comparable (but not identical) quantizations, and TensorRT-LLM’s `AWQ INT4` is implemented differently from llama.cpp’s `q4_k_m`
|
||||
|
||||
### Max RAM Usage
|
||||
|
||||
| Max RAM utilization | TensorRT-LLM | Llama.cpp | % Difference |
|
||||
| ---------------------------- | ------------ | --------- | ---------------- |
|
||||
| GeForce RTX 4090 desktop GPU | ✅ 4.98 | 7.11 | ✅ 29.88% less |
|
||||
| GeForce RTX 3090 desktop GPU | ✅ 0.98 | 2.60 | ✅ 62.41% less |
|
||||
| GeForce RTX 4070 Laptop GPU | ✅ 1.04 | 4.44 | ✅ 76.55%% less |
|
||||
| GeForce RTX 4090 eGPU | ✅ 4.11 | 5.38 | ✅ 23.61% less |
|
||||
|
||||
TensorRT-LLM uses a lot less Max RAM vs. llama.cpp on slower connection (PCIe 3.0 or Thunderbolt 3) due to better memory management and efficient delegation to VRAM. On faster connection, it’s at least equal to llama.cpp.
|
||||
|
||||
### Compiled Model Size and Number of Files
|
||||
- Contrary to popular belief, TensorRT-LLM prebuilt models turned out to not be that huge
|
||||
- Mistral 7b int4 was actually 25% smaller in TensorRT-LLM, at 3.05gb vs. 4.06gb
|
||||
- Note: These are approximate comparisons, as TensorRT-LLM’s AWQ INT4 is implemented differently from llama.cpp’s q4_k_m
|
||||
- The bigger takeaway is that the Compiled model sizes are roughly in the same ballpark, while the number of files for TensorRT-LLM is 7x the GGUF number of files.
|
||||
|
||||
| Model size (Lower is better) | TensorRT-LLM AWQ int4 | Llama.cpp GGUF Q4 | % Difference |
|
||||
| ---------------------------- | --------------------- | ----------------- | ----------------- |
|
||||
| Mistral 7B | ✅ 3.05GB | 4.06GB | ✅ 24.88% smaller |
|
||||
|
||||
### Convenience
|
||||
- Llama.cpp still wins on cross-platform versatility and convenience of a “compile once, run everywhere” approach
|
||||
- TensorRT-LLM still requires compilation to specific OS and architecture, though this could be solved by pre-compiling and publishing models on [Nvidia's NGC Model Catalog](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-foundation/collections/codellama)
|
||||
|
||||
### Accessibility
|
||||
- Llama.cpp unsurprisingly beats TensorRT-LLM in terms of accessibility
|
||||
- TensorRT-LLM does not support older NVIDIA GPUs and won’t work well on smaller VRAM cards (e.g. 2-4gb VRAM)
|
||||
|
||||
## Final Notes
|
||||
|
||||
Our benchmarking is not perfect. We evaluated over a dozen tools ([llmperf](https://github.com/ray-project/llmperf), [psutil](https://pypi.org/project/psutil/), [gpustat](https://github.com/wookayin/gpustat), native utilities, and more) and found that everyone measures TPS, common metrics differently. We eventually settled on using our own tools in Jan, which are consistent across any inference engine and hardware. As for runtime parameters, we went with default settings, likely representing the typical user experience.
|
||||
|
||||
We also did not overclock for this benchmark , as it is not a default setting for most users. But we've measured in our tests that TensorRT-LLM can go even faster with a few tweaks. We see this as a pretty exciting future direction.
|
||||
|
||||
<Aside type="note" >
|
||||
**How to Overclock:** We found an additional **15% increase in performance** with TensorRT-LLM by:
|
||||
- Enabling [XMP](https://www.intel.com/content/www/us/en/gaming/extreme-memory-profile-xmp.html)
|
||||
- Overclocking RAM bus speed in BIOS from `3600` to `5600`.
|
||||
</Aside>
|
||||
|
||||
We're also publishing the underlying [raw experimental data](https://drive.google.com/file/d/1rDwd8XD8erKt0EgIKqOBidv8LsCO6lef/view?usp=sharing), and would encourage the community to scrutinize and help us improve.
|
||||
|
||||
Special thanks to Asli Sabanci Demiroz, Annamalai Chockalingam, Jordan Dodge from Nvidia, and Georgi Gerganov from llama.cpp for feedback, review and suggestions.
|
||||
@ -1,149 +0,0 @@
|
||||
---
|
||||
title: 'Bitdefender False Positive Flag'
|
||||
description: "10th January 2024, Jan's 0.4.4 Release on Windows triggered Bitdefender to incorrectly flag it as infected with Gen:Variant.Tedy.258323, leading to automatic quarantine warnings on users' computers."
|
||||
date: 2024-01-10
|
||||
tags: postmortem, bitdefender
|
||||
categories: building-jan
|
||||
keywords:
|
||||
[
|
||||
postmortem,
|
||||
bitdefender,
|
||||
false positive,
|
||||
antivirus,
|
||||
jan,
|
||||
nitro,
|
||||
incident,
|
||||
incident response,
|
||||
supply chain security,
|
||||
user communication,
|
||||
documentation,
|
||||
antivirus compatibility,
|
||||
cross-platform testing,
|
||||
proactive incident response,
|
||||
user education,
|
||||
lessons learned,
|
||||
]
|
||||
---
|
||||
|
||||
import CTABlog from '@/components/Blog/CTABlog.astro';
|
||||
|
||||
|
||||
# Bitdefender False Positive Flag
|
||||
|
||||
Following the recent incident related to Jan version 0.4.4 triggering Bitdefender on Windows with Gen:Variant.Tedy.258323 on January 10, 2024, we wanted to provide a comprehensive postmortem and outline the necessary follow-up actions.
|
||||
|
||||
## Incident Overview
|
||||
|
||||
### Bug Description
|
||||
|
||||
Jan 0.4.4 installation on Windows triggered Bitdefender to flag it as infected with Gen:Variant.Tedy.258323, leading to automatic quarantine.
|
||||
|
||||
### Affected Antivirus
|
||||
|
||||
- McAfee / Microsoft Defender was unaffected
|
||||
- Bitdefender consistently flagged the issue.
|
||||
|
||||
### Incident Timeline
|
||||
|
||||
- _10 Jan, 2:18 am SGT:_ Hawke flags up Malware antivirus errors for 0.4.4 installation on Windows computers.
|
||||
- _10 Jan, 2:21 am SGT:_ @0xSage responds in Discord.
|
||||
- _10 Jan, 2:35 am SGT:_ Hawke confirms multiple people have experienced this error on fresh installs.
|
||||
- _10 Jan, 2:41 am SGT:_ @louis-jan and @dan-jan revert 0.4.4 out of an abundance of caution.
|
||||
- _Incident ongoing:_ To triage and investigate the next day.
|
||||
- _10 Jan, 11:36 am SGT:_ @Hien has investigated all versions of Nitro and conducted scans using Bitdefender. Only the 2 latest versions raised warnings (0.2.7, 0.2.8).
|
||||
- _10 Jan, 12:44 pm SGT:_ @Hien tested again for the 0.2.6 and suggested using 0.2.6 for now, the 2 remaining Nitro version (0.2.7, 0.2.8) will under further investigation.
|
||||
- The team started testing on the fixed build.
|
||||
- _10 Jan, 3:22 pm SGT:_ Diagnosis found that it's most likely a false positive. @Hien has only found a solution by attempting to build Nitro Windows CPU on a GitHub-hosted runner and hasn't identified the root cause yet.
|
||||
- _10 Jan, 5:24 pm SGT:_ @Hien testing two scenarios and still trying to understand the workings of Bitdefender.
|
||||
- _11 Jan, 5:46 pm SGT:_ Postmortem meeting
|
||||
|
||||
## Investigation Update
|
||||
|
||||
- @Hien has investigated all versions of Nitro and conducted scans using Bitdefender. and only the 2 latest versions raised warnings from Bitdefender. Nitro 0.2.6, which is the highest version without the issue, was tested again, and it no longer triggers a warning from Bitdefender.
|
||||
- We have observed that Nitro versions up to 0.2.6 remain unaffected. However, Bitdefender flags versions 0.2.7 and 0.2.8 as infected, leading to the deletion. In order to proceed with the current release, Hien suggests downgrading Nitro to version 0.2.6 and conducting tests with this version. Simultaneously, he will investigate why Bitdefender is flagging versions 0.2.7 and 0.2.8.
|
||||
- It's essential to note that between versions 0.2.6, 0.2.7, and 0.2.8, only minor changes were made, which should not trigger a malicious code warning. We can refer to the changelog between 0.2.7 and 0.2.8 to pinpoint these changes.
|
||||
- Our primary message is to convey that we did not introduce malicious code into Jan (indicating a false positive), and the investigation aims to understand the root cause behind Bitdefender flagging versions 0.2.7 and 0.2.8.
|
||||
- The current diagnosis looks like a false positive but it's still under investigation. Reference link: [here](https://stackoverflow.com/questions/75886428/fake-positive-bit-defender-problem-genvariant-tedy-304469), [here](https://stackoverflow.com/questions/58010466/bitdefender-detects-my-console-application-as-genvariant-ursu-56053), and [here](https://www.cisa.gov/sites/default/files/2023-06/mar-10365227.r1.v1.clear_.pdf).
|
||||
- @Hien testing two scenarios and still trying to understand the workings of Bitdefender. Still under investigation: is the issue with the code or the CI?
|
||||
- In Case 1, using the same CI agent for tags 0.2.6 and 0.2.8, after PRs by Alan and myself, Bitdefender flagged the Nitro CPU binary build. Naturally, one would conclude this is due to the code.
|
||||
- However, I proceeded with a further experiment: for the 0.2.8 code, instead of using our CI agent, I used a GitHub hosted agent. This time, Bitdefender did not flag our binary build.
|
||||
- We've identified the Bitdefender warning was not an attack. There is no malicious code
|
||||
- We've isolated the event to originate from a CI agent, which resulted in a BitDefender false positive alert.
|
||||
|
||||
## Follow-ups and Action Items
|
||||
|
||||
1. **Reproduce Bitdefender Flag in Controlled Environment [Done]:**
|
||||
|
||||
- _Objective:_ To replicate the issue in a controlled environment to understand the triggers and specifics of Bitdefender's detection.
|
||||
|
||||
2. **Investigate Malicious Code or False Positive:**
|
||||
|
||||
- _Objective:_ Determine whether the flagged issue is a result of actual malicious code or a false positive. If it's a false positive, work towards resolution while communicating with Bitdefender.
|
||||
|
||||
3. **Supply Chain Attack Assessment:**
|
||||
|
||||
- _Objective:_ Evaluate the possibility of a supply chain attack. Investigate whether the Nitro 0.4.4 distribution was compromised or tampered with during the release process.
|
||||
|
||||
4. **Testing after the Hotfix:**
|
||||
|
||||
- _Objective:_ In addition to verifying the issue after the fix, it is essential to conduct comprehensive testing across related areas, ensuring compatibility across different operating systems and antivirus software (latest version / free version only).
|
||||
|
||||
5. **Process Improvement for Future Releases:**
|
||||
|
||||
- _Objective:_ Identify and implement improvements to our release process to prevent similar incidents in the future. This may include enhanced testing procedures, code analysis, and collaboration with antivirus software providers during the pre-release phase. Additionally, we should add verifying the latest antivirus software in the release checklist.
|
||||
|
||||
6. **Documentation of Tested Antivirus Versions:**
|
||||
- _Objective:_ Create a document that outlines the testing conducted, including a matrix that correlates Jan versions with the tested antivirus versions.
|
||||
- _Sample list:_ for consideration purpose
|
||||
- Bitdefender
|
||||
- McAfee
|
||||
- Avira
|
||||
- Kaspersky
|
||||
- Norton
|
||||
- Microsoft defender
|
||||
- AVG
|
||||
- TotalAV
|
||||
|
||||
## Next Steps
|
||||
|
||||
- The team should follow up on each action item with clear ownership priority, and deadlines.
|
||||
- Communicate progress transparently with the community and clients through appropriate channels. If any insights or suggestions, share them within the dedicated channels.
|
||||
- Update internal documentation and procedures based on the lessons learned from this incident.
|
||||
|
||||
## Lessons Learned
|
||||
|
||||
1. **Antivirus Compatibility Awareness:**
|
||||
|
||||
- _Observation:_ The incident underscored the significance of recognizing and testing for antivirus compatibility, particularly with widely-used solutions like Bitdefender.
|
||||
- _Lesson Learned:_ In the future, we will integrate comprehensive checks for compatibility with various antivirus software, including both antivirus and "Malicious Code Detection," into our CI or QA checklist. This proactive measure aims to minimize false positive detections during the release and testing processes.
|
||||
|
||||
2. **Cross-Platform Testing:**
|
||||
|
||||
- _Observation:_ The problem did not occur on MacOS and Linux systems, implying a potential oversight in cross-platform testing during our release procedures.
|
||||
- _Lesson Learned:_ Clarification — This observation is not directly related to antivirus testing. Instead, it underscores the necessity to improve our testing protocols, encompassing multiple operating systems. This ensures a thorough evaluation of potential issues on diverse platforms, considering the various antivirus software and differences in architectures on Mac and Linux systems.
|
||||
|
||||
3. **User Communication and Documentation:**
|
||||
|
||||
- _Observation:_ Due to the timely response from Nicole, who was still active on Discord and Github at 2 am, this quick response facilitated our ability to assess the impact accurately.
|
||||
- _Lesson Learned:_ While our communication with users was effective in this instance, it was mainly due to Nicole's presence during the incident. To improve our overall response capability, we should prioritize "24/7 rapid triage and response." This involves ensuring continuous availability or establishing a reliable rotation of team members for swift user communication and issue documentation, further enhancing our incident response efficiency.
|
||||
|
||||
4. **Proactive Incident Response:**
|
||||
|
||||
- _Observation:_ The incident response, while involving a prompt version rollback, experienced a slight delay due to the release occurring at midnight. This delay postponed the initiation of the investigation until the next working hours.
|
||||
- _Lesson Learned:_ Recognizing the importance of swift incident response, particularly in time-sensitive situations, we acknowledge that releasing updates during off-hours can impact the immediacy of our actions. Moving forward, we will strive to optimize our release schedules to minimize delays and ensure that investigations can commence promptly regardless of the time of day. This may involve considering alternative release windows or implementing automated responses to critical incidents, ensuring a more proactive and timely resolution.
|
||||
|
||||
5. **Supply Chain Security Measures:**
|
||||
|
||||
- _Observation:_ While the incident prompted consideration of a potential supply chain attack, it's crucial to emphasize that this was not the case. Nonetheless, the incident underscored the importance of reviewing our supply chain security measures.
|
||||
- _Lesson Learned:_ Going forward, we should strengthen supply chain security by introducing additional verification steps to uphold the integrity of our release process. Collaborating with distribution channels is essential for enhancing security checks and ensuring a robust supply chain.
|
||||
- _Longer-term:_ Exploring options for checking Jan for malicious code and incorporating antivirus as part of our CI/CD pipeline should be considered for a more comprehensive and proactive approach.
|
||||
|
||||
6. **User Education on False Positives:**
|
||||
- _Observation:_ Users reported Bitdefender automatically "disinfecting" the flagged Nitro version without allowing any user actions.
|
||||
- _Lesson Learned:_ Educate users about the possibility of false positives and guide them on how to whitelist or report such incidents to their antivirus provider (if possible). Provide clear communication on steps users can take in such situations.
|
||||
|
||||
These lessons learned will serve as a foundation for refining our processes and ensuring a more resilient release and incident response framework in the future. Continuous improvement is key to maintaining the reliability and security of our software.
|
||||
|
||||
Thank you for your dedication and cooperation in resolving this matter promptly.
|
||||
|
||||
<CTABlog />
|
||||
@ -1,116 +0,0 @@
|
||||
---
|
||||
title: "The Invisible Moat around Open-Source LLM"
|
||||
description: "Uncover the pivotal role of data ownership in training the next iteration of LLM."
|
||||
tags: OpenAI has a moat, Catastrophic forgetting, ChatGPT
|
||||
date: 2024-03-25
|
||||
unlisted: true
|
||||
categories: research
|
||||
---
|
||||
|
||||
import CTABlog from '@/components/Blog/CTABlog.astro';
|
||||
|
||||
import catastrophicDemo from '@/assets/blog/catastrophic-demo.png';
|
||||
import gradientDecent from '@/assets/blog/gradient-decent.gif';
|
||||
import replay from '@/assets/blog/replay.png';
|
||||
import openchatBench0106 from '@/assets/blog/openchat-bench-0106.png';
|
||||
|
||||
|
||||
# The Invisible Moat around Open-Source LLM
|
||||
|
||||
In the crowded AI landscape, OpenAI's ChatGPT stands out, not just for its capabilities but for its unique access to the pre-trained dataset. This post explores the vital role of data in maintaining a competitive edge, focusing on OpenAI's strategic advantage through data ownership.
|
||||
|
||||
## Data: The Secret Weapon
|
||||
OpenAI, with ChatGPT, has carved a distinct advantage. By harnessing user interactions, it gains invaluable insights into diverse use cases, enabling precise model refinements. The cornerstone of this advantage lies in the "pre-trained dataset." This treasure trove of data empowers OpenAI to cater to specific needs, ensuring sustained improvement and differentiation.
|
||||
|
||||
## The rise of the opensource
|
||||
|
||||
```
|
||||
- How they/Mistral/Llama make money?
|
||||
-> around having pretrained data -> finetuning
|
||||
First para:
|
||||
Rise of Open Source LLMs like Mistral, Llama2, Llama3
|
||||
People think they don't have a moat = everything is open source
|
||||
Second para:
|
||||
We actually think these guys have an "invisible moat"
|
||||
Pre-training data is not released, and makes a huge difference in fine-tuning efficacy
|
||||
```
|
||||
|
||||
### Why pretrained data is important?
|
||||
|
||||
> *Owning the pre-trained dataset is crucial as it represents the original distribution.*
|
||||
Access to the pre-trained dataset acts as a master key to address the critical issue of ["Catastrophic forgetting"](https://en.wikipedia.org/wiki/Catastrophic_interference) in Language Learning Models (LLMs). This phenomenon describes how LLMs lose hold of prior knowledge upon learning new information. Access to the foundational dataset allows for effective fine-tuning, balancing the introduction of new data with the retention of existing knowledge.
|
||||
|
||||
<img src={catastrophicDemo.src} alt="Catastrophic forgetting" />
|
||||
|
||||
**Figure 1.** Demonstrates the catastrophic forgetting issue: without mixing datasets, AI overfits on new tasks, impairing normal communication.
|
||||
|
||||
### Illustrating Catastrophic Forgetting
|
||||
|
||||
```
|
||||
What is fine-tuning
|
||||
Process of Finetuning (pretrain, instruct, finetune)
|
||||
Fine-tuning datasets
|
||||
Risk of catastrophic forgetting
|
||||
"Why is Pre-trained data important?"
|
||||
What is pre-training dataset
|
||||
How does fine-tuning with pre-training dataset differ from when you don't have it
|
||||
How does it avoid catastrophic forgetting
|
||||
```
|
||||
|
||||
Catastrophic forgetting can be visualized as a ball in a multidimensional landscape, where moving towards new knowledge risks losing grasp on the old.
|
||||
Pre-trained data acts as a map, guiding fine-tuning in a way that incorporates new information while safeguarding existing knowledge.
|
||||
|
||||
<img src={gradientDecent.src} alt="Gradient decent" />
|
||||
|
||||
**Figure 2.** [Gradient decent demonstration](https://en.wikipedia.org/wiki/Gradient_descent)
|
||||
|
||||
### Smoothing Distribution Shifts
|
||||
|
||||
As described above, with the mixture of the pre-trained dataset ensures smoother distribution shifts when introducing new information, as it embodies a comprehensive spectrum of prior knowledge.
|
||||
|
||||
This continuity in knowledge transition helps in maintaining the robustness of the model against sudden changes, akin to providing a more gradual learning curve where the new information is incrementally integrated with the existing knowledge base.
|
||||
|
||||
This concept is supported by the [EleutherAI's research](https://arxiv.org/abs/2403.08763) highlighting the importance of how tasks are sequenced in the learning process, suggesting that introducing dissimilar tasks early on can expand the network's capacity for new information.
|
||||
|
||||
**Table 1.** Final results for English-only 405M parameter models trained with different replay amounts show models with more replay perform better in balancing learning and forgetting (measured as AVG Loss). Notably, just 1% mix with a pre-trained dataset significantly lowers AVG loss, effectively shifting model knowledge from English (the Pile) to German.
|
||||
|
||||
<img src={replay.src} alt="Replay method" />
|
||||
|
||||
*Note:* **Replay** is the method involves combining the training dataset from the pre-trained model with new task datasets.
|
||||
|
||||
### Acting as a Noise Mask
|
||||
|
||||
The pre-trained data can also serve as a form of "noise masking", similar to techniques used in training [early computer vision models](https://arxiv.org/abs/1911.04252).
|
||||
|
||||
This approach introduces a level of ["noise"](https://arxiv.org/abs/2310.05914) during training, which can prevent the model from overfitting to the new dataset. By retaining a mix of original and new data, the model is exposed to a broader range of scenarios, enhancing its generalization capabilities and robustness across tasks.
|
||||
|
||||
## Solutions
|
||||
|
||||
### Overwhelming approach
|
||||
|
||||
Overcoming these challenges requires a balanced approach. One partial method involves inundating the model with extensive, curated data, allowing for comprehensive fine-tuning. While effective, this approach demands significant computational resources, a comprehensive filtering process for low-quality inputs, and an extraordinarily high cost associated with gathering millions of high-quality responses.
|
||||
|
||||
In the open-source community, 2 notable examples of fine-tuning with Mistral as a base model on large datasets collected from top-rated GPT-4 and human responses demonstrate a distribution shift that enhances model performance, including [OpenChat](https://huggingface.co/openchat/openchat-3.5-0106) and [Hermes-Pro](https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B).
|
||||
|
||||
<img src={openchatBench0106.src} alt="Openchat results" />
|
||||
|
||||
**Figure 2.** After fine-tuning with a large amount of data samples, the model's performance improved, outperforming ChatGPT and Grok-1 in some benchmarks.
|
||||
|
||||
### Fully open source model
|
||||
|
||||
- Example: Dolma + olma from allenai
|
||||
|
||||
## Conclusion
|
||||
|
||||
The ownership and strategic use of pre-trained data serve as an invisible moat. It not only enables the tackling of complex challenges like catastrophic forgetting but also provides a baseline for continuous, targeted improvements. Although there is a solution to decentralize, the cost remains reasonably high.
|
||||
|
||||
Fully open pretrained + open weight
|
||||
|
||||
## Reference
|
||||
- [Catastrophic forgetting](https://arxiv.org/abs/2308.08747)
|
||||
- [Simple and Scalable Strategies to Continually Pre-train Large Language Models](https://arxiv.org/abs/2403.08763)
|
||||
- [Gradient descent](https://en.wikipedia.org/wiki/Gradient_descent)
|
||||
- [Neftune](https://arxiv.org/abs/2310.05914)
|
||||
- [Self-training with Noisy Student improves ImageNet classification](https://arxiv.org/abs/1911.04252)
|
||||
|
||||
<CTABlog />
|
||||
@ -1,141 +0,0 @@
|
||||
---
|
||||
title: "Run DeepSeek R1 locally on your device (Beginner-Friendly Guide)"
|
||||
description: "A straightforward guide to running DeepSeek R1 locally regardless of your background."
|
||||
tags: DeepSeek, R1, local AI, Jan, GGUF, Qwen, Llama
|
||||
categories: guides
|
||||
date: 2025-01-31
|
||||
ogImage: assets/deepseek-r1-locally-jan.jpg
|
||||
twitter:
|
||||
card: summary_large_image
|
||||
site: "@jandotai"
|
||||
title: "Run DeepSeek R1 locally on your device (Beginner-Friendly Guide)"
|
||||
description: "A straightforward guide to running DeepSeek R1 locally regardless of your background."
|
||||
image: assets/deepseek-r1-locally-jan.jpg
|
||||
---
|
||||
|
||||
import CTABlog from '@/components/Blog/CTABlog.astro';
|
||||
import { Aside } from '@astrojs/starlight/components'
|
||||
|
||||
|
||||
import deepseekR1LocallyJan from '@/assets/blog/deepseek-r1-locally-jan.jpg';
|
||||
import downloadJan from '@/assets/blog/download-jan.jpg';
|
||||
import janLibraryDeepseekR1 from '@/assets/blog/jan-library-deepseek-r1.jpg';
|
||||
import janHubDeepseekR1 from '@/assets/blog/jan-hub-deepseek-r1.jpg';
|
||||
import janRunsDeepseekR1Distills from '@/assets/blog/jan-runs-deepseek-r1-distills.jpg';
|
||||
|
||||
|
||||
# Run DeepSeek R1 locally on your device (Beginner-Friendly Guide)
|
||||
|
||||
<img src={deepseekR1LocallyJan.src} alt="DeepSeek R1 running locally in Jan AI interface, showing the chat interface and model settings" />
|
||||
|
||||
DeepSeek R1 is one of the best open-source models in the market right now, and you can run DeepSeek R1 on your own computer!
|
||||
|
||||
<Aside type="note">
|
||||
New to running AI models locally? Check out the [guide on running AI models locally](/blog/run-ai-models-locally) first. It covers essential concepts that will help you better understand this DeepSeek R1 guide.
|
||||
</Aside>
|
||||
|
||||
DeepSeek R1 requires data-center level computers to run at its full potential, and we'll use a smaller version that works great on regular computers.
|
||||
|
||||
Why use an optimized version?
|
||||
- Efficient performance on standard hardware
|
||||
- Faster download and initialization
|
||||
- Optimized storage requirements
|
||||
- Maintains most of the original model's capabilities
|
||||
|
||||
## Quick Steps at a Glance
|
||||
1. Download [Jan](https://jan.ai/)
|
||||
2. Select a model version
|
||||
3. Choose settings
|
||||
4. Set up the prompt template & start using DeepSeek R1
|
||||
|
||||
Let's walk through each step with detailed instructions.
|
||||
|
||||
## Step 1: Download Jan
|
||||
[Jan](https://jan.ai/) is an open-source application that enables you to run AI models locally. It's available for Windows, Mac, and Linux. For beginners, Jan is the best choice to get started.
|
||||
|
||||
<img src={downloadJan.src} alt="Jan AI interface, showing the download button" />
|
||||
|
||||
1. Visit [jan.ai](https://jan.ai)
|
||||
2. Download the appropriate version for your operating system
|
||||
3. Install the app
|
||||
|
||||
## Step 2: Choose Your DeepSeek R1 Version
|
||||
|
||||
To run AI models like DeepSeek R1 on your computer, you'll need something called VRAM (Video Memory). Think of VRAM as your computer's special memory for handling complex tasks like gaming or, in our case, running AI models. It's different from regular RAM - VRAM is part of your graphics card (GPU).
|
||||
|
||||
<Aside type="note">
|
||||
Running AI models locally is like running a very sophisticated video game - it needs dedicated memory to process all the AI's "thinking." The more VRAM you have, the larger and more capable AI models you can run.
|
||||
</Aside>
|
||||
|
||||
Let's first check how much VRAM your computer has. Don't worry if it's not much - DeepSeek R1 has versions for all kinds of computers!
|
||||
|
||||
Finding your VRAM is simple:
|
||||
- On Windows: Press `Windows + R`, type `dxdiag`, hit Enter, and look under the "Display" tab
|
||||
- On Mac: Click the Apple menu, select "About This Mac", then "More Info", and check under "Graphics/Displays"
|
||||
- On Linux: Open Terminal and type `nvidia-smi` for NVIDIA GPUs, or `lspci -v | grep -i vga` for other graphics cards
|
||||
|
||||
<Aside type="tip">
|
||||
**No dedicated graphics card?** That's okay! You can still run the smaller versions of DeepSeek R1. They're specially optimized to work on computers with basic graphics capabilities.
|
||||
</Aside>
|
||||
|
||||
Once you know your VRAM, here's what version of DeepSeek R1 will work best for you. If you have:
|
||||
- 6GB VRAM: Go for the 1.5B version - it's fast and efficient
|
||||
- 8GB VRAM: You can run the 7B or 8B versions, which offer great capabilities
|
||||
- 16GB or more VRAM: You have access to the larger models with enhanced features
|
||||
|
||||
Available versions and basic requirements for DeepSeek R1 distills:
|
||||
|
||||
| Version | Model Link | Required VRAM |
|
||||
|---------|------------|---------------|
|
||||
| Qwen 1.5B | [DeepSeek-R1-Distill-Qwen-1.5B-GGUF](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF) | 6GB+ |
|
||||
| Qwen 7B | [DeepSeek-R1-Distill-Qwen-7B-GGUF](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | 8GB+ |
|
||||
| Llama 8B | [DeepSeek-R1-Distill-Llama-8B-GGUF](https://huggingface.co/unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF) | 8GB+ |
|
||||
| Qwen 14B | [DeepSeek-R1-Distill-Qwen-14B-GGUF](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-14B-GGUF) | 16GB+ |
|
||||
| Qwen 32B | [DeepSeek-R1-Distill-Qwen-32B-GGUF](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-32B-GGUF) | 16GB+ |
|
||||
| Llama 70B | [DeepSeek-R1-Distill-Llama-70B-GGUF](https://huggingface.co/unsloth/DeepSeek-R1-Distill-Llama-70B-GGUF) | 48GB+ |
|
||||
|
||||
To download your chosen model:
|
||||
|
||||
Launch Jan and navigate to Jan Hub using the sidebar
|
||||
|
||||
<img src={janLibraryDeepseekR1.src} alt="Jan AI interface, showing the model library" />
|
||||
|
||||
3. Input the model link in this field:
|
||||
|
||||
<img src={janHubDeepseekR1.src} alt="Jan AI interface, showing the model link input field" />
|
||||
|
||||
## Step 3: Configure Model Settings
|
||||
When configuring your model, you'll encounter quantization options:
|
||||
|
||||
<Aside type="tip">
|
||||
Quantization balances performance and resource usage:
|
||||
- **Q4:** Recommended for most users - optimal balance of efficiency and quality
|
||||
- **Q8:** Higher precision but requires more computational resources
|
||||
</Aside>
|
||||
|
||||
## Step 4: Configure Prompt Template
|
||||
Final configuration step:
|
||||
|
||||
1. Access Model Settings via the sidebar
|
||||
2. Locate the Prompt Template configuration
|
||||
3. Use this specific format:
|
||||
|
||||
<Aside type="caution">
|
||||
```
|
||||
<|User|>{prompt}<|Assistant|>
|
||||
```
|
||||
</Aside>
|
||||
|
||||
This template is for proper communication between you and the model.
|
||||
|
||||
You're now ready to interact with DeepSeek R1:
|
||||
|
||||
<img src={janRunsDeepseekR1Distills.src} alt="Jan interface, showing DeepSeek R1 running locally" />
|
||||
|
||||
## Need Assistance?
|
||||
|
||||
<Aside type="note">
|
||||
Join our [Discord community](https://discord.gg/Exe46xPMbK) for support and discussions about running AI models locally.
|
||||
</Aside>
|
||||
|
||||
<CTABlog />
|
||||
@ -1,125 +0,0 @@
|
||||
---
|
||||
title: "Offline ChatGPT: You can't run ChatGPT offline, do this instead"
|
||||
description: "Learn how to use AI offline with Jan - a free, open-source alternative to ChatGPT that works 100% offline on your computer."
|
||||
tags: AI, ChatGPT alternative, offline AI, Jan, local AI, privacy
|
||||
categories: guides
|
||||
date: 2025-02-08
|
||||
ogImage: _assets/offline-chatgpt-alternatives-jan.jpg
|
||||
twitter:
|
||||
card: summary_large_image
|
||||
site: "@jandotai"
|
||||
title: "Offline ChatGPT: You can't run ChatGPT offline, do this instead"
|
||||
description: "Want to use ChatGPT offline? Learn how to run AI models locally with Jan - free, open-source, and works without internet."
|
||||
image: _assets/offline-chatgpt-alternatives-jan.jpg
|
||||
---
|
||||
|
||||
import CTABlog from '@/components/Blog/CTABlog.astro';
|
||||
import { Aside } from '@astrojs/starlight/components'
|
||||
|
||||
|
||||
import offlineChatgptAlternativeAiWithoutInternet from '@/assets/blog/offline-chatgpt-alternative-ai-without-internet.jpg';
|
||||
|
||||
|
||||
# Offline ChatGPT: You can't run ChatGPT offline, do this instead
|
||||
|
||||
ChatGPT is a cloud-based service that requires internet access. However, it's not the only way to use AI. You can run AI models offline on your device with [Jan](https://jan.ai/). It's completely free, open-source, and gives you 100% offline capability. You can even use AI on a plane!
|
||||
|
||||
<Aside type="tip">
|
||||
**Quick Summary:**
|
||||
- ChatGPT always needs internet - it can't run offline
|
||||
- Jan lets you run AI models 100% offline on your computer
|
||||
- It's free and open-source
|
||||
- Works on Mac, Windows, and Linux
|
||||
</Aside>
|
||||
|
||||
## Jan as an offline ChatGPT alternative
|
||||
|
||||
<img src={offlineChatgptAlternativeAiWithoutInternet.src} alt="Use Jan to chat with AI models without internet access" />
|
||||
*Jan lets you use AI offline - no internet connection needed*
|
||||
|
||||
Here's how to get started with offline AI in 3 simple steps:
|
||||
|
||||
### 1. Download Jan
|
||||
|
||||
Go to [jan.ai](https://jan.ai) and download the version for your computer (Mac, Windows, or Linux). It's completely free.
|
||||
|
||||
|
||||
|
||||
### 2. Download an AI model
|
||||
|
||||
You'll need an AI model to use AI offline, so download a model from Jan. Once it's on your computer, you don't need internet anymore.
|
||||
|
||||
|
||||
*Select an AI model that matches your needs and computer capabilities*
|
||||
|
||||
<Aside type="tip">
|
||||
**Which model should you choose?**
|
||||
- For most computers: Try Mistral 7B or DeepSeek - they're similar to ChatGPT 3.5
|
||||
- For older computers: Use smaller 3B models
|
||||
- For gaming PCs: You can try larger 13B models
|
||||
|
||||
Don't worry about choosing - Jan will automatically recommend models that work well on your computer.
|
||||
</Aside>
|
||||
|
||||
### 3. Start using AI offline
|
||||
|
||||
|
||||
*Use Jan's clean interface to chat with AI - no internet required*
|
||||
|
||||
Once downloaded, you can use AI anywhere, anytime:
|
||||
|
||||
- Chat like you do with ChatGPT
|
||||
- Work on documents offline
|
||||
- Get coding help without internet
|
||||
- Keep your conversations private
|
||||
- Use AI even when servers are down
|
||||
|
||||
## How to chat with your docs in Jan?
|
||||
|
||||
To chat with your docs in Jan, you need to activate experimental mode.
|
||||
|
||||
|
||||
*Turn on experimental mode in settings to chat with your docs*
|
||||
|
||||
After activating experimental mode, simply add your files and ask questions about them.
|
||||
|
||||
|
||||
*Chat with your documents privately - no internet needed*
|
||||
|
||||
I did this for you and got a reply from a 7B parameter model. If you'd like to learn what "7B" means and understand other local AI terms, check our [guide on running AI models locally](/blog/run-ai-models-locally).
|
||||
|
||||
A response from AI, Qwen2.5 7B Instruct Q4:
|
||||
|
||||
`This document appears to be about the benefits and advantages of running artificial intelligence (AI) models locally on your device rather than using cloud-based or remote AI services. The key points it highlights include data privacy, offline functionality, freedom from paywalls and restrictions, and giving users full control over their AI models. Additionally, the text mentions that local AI is becoming a new trend and provides a link to a guide for beginners who want to learn more about this topic.`
|
||||
|
||||
Local AI makes possible offline AI use, so Jan is going to be your first step to get started.
|
||||
|
||||
## Why choose Jan over ChatGPT?
|
||||
|
||||
1. **True Offline Use:** Unlike ChatGPT, Jan works without internet
|
||||
2. **100% Private:** Your data never leaves your computer
|
||||
3. **Free Forever:** No subscriptions or API costs
|
||||
4. **No Server Issues:** No more "ChatGPT is at capacity"
|
||||
5. **Your Choice of Models:** Use newer models as they come out
|
||||
|
||||
**"Is it really free? What's the catch?"**
|
||||
Yes, it's completely free and open source. Jan is built by developers who believe in making AI accessible to everyone.
|
||||
|
||||
**"How does it compare to ChatGPT?"**
|
||||
Modern open-source models like DeepSeek and Mistral are very capable. While they might not match GPT-4, they're perfect for most tasks and getting better every month.
|
||||
|
||||
**"Do I need a powerful computer?"**
|
||||
If your computer is from the last 5 years, it will likely work fine. You need about 8GB of RAM and 10GB of free space for comfortable usage.
|
||||
|
||||
**"What about my privacy?"**
|
||||
Everything stays on your computer. Your conversations, documents, and data never leave your device unless you choose to share them.
|
||||
|
||||
Want to learn more about the technical side? Check our detailed [guide on running AI models locally](/blog/run-ai-models-locally). It's not required to [use AI offline](https://jan.ai/) but helps understand how it all works.
|
||||
|
||||
## Need help?
|
||||
|
||||
<Aside type="note">
|
||||
[Join our Discord community](https://discord.gg/Exe46xPMbK) for support and tips on using Jan as your offline ChatGPT alternative.
|
||||
</Aside>
|
||||
|
||||
<CTABlog />
|
||||
@ -1,131 +0,0 @@
|
||||
---
|
||||
title: "Best Settings to Run Qwen3-30B-A3B Locally"
|
||||
description: "If you're running Qwen3-30B-A3B locally, don't guess your way through the settings. This guide tells you what actually works based on Qwen's own documentation and what we've seen hold up in practice."
|
||||
tags: Qwen3, local AI, model settings, Jan, offline AI
|
||||
categories: guides
|
||||
date: 2025-05-10
|
||||
ogImage: assets/images/general/qwen3-30b-settings.jpg
|
||||
---
|
||||
|
||||
import CTABlog from '@/components/Blog/CTABlog.astro';
|
||||
import { Aside } from '@astrojs/starlight/components'
|
||||
|
||||
|
||||
import qwen3SettingsJanAi from '@/assets/blog/qwen3-settings-jan-ai.jpeg';
|
||||
import qwen3InJanHub from '@/assets/blog/qwen3-in-jan-hub.jpeg';
|
||||
import qwen3SettingsInJan from '@/assets/blog/qwen3-settings-in-jan.jpeg';
|
||||
|
||||
|
||||
# Best Settings to Run Qwen3-30B-A3B Locally
|
||||
|
||||
If you're running Qwen3-30B-A3B locally, don't guess your way through the settings. This guide tells you what actually works based on Qwen's own documentation and what we've seen hold up in practice.
|
||||
|
||||
<Aside type="tip">
|
||||
**Quick Summary:**
|
||||
- Qwen3-30B-A3B has two modes: thinking and non-thinking
|
||||
- Each mode needs different generation settings
|
||||
- Greedy decoding breaks thinking mode - avoid it
|
||||
- Don't log <think> blocks in chat history
|
||||
</Aside>
|
||||
|
||||
Qwen3 comes with a unique toggle: `enable_thinking`. When it's on, the model "thinks", it breaks down problems, reasons step-by-step, and wraps part of its output in a `<think>...</think>` block. When it's off, the model skips all that and just gives you an answer.
|
||||
|
||||
That changes how you configure it.
|
||||
|
||||
---
|
||||
|
||||
### Thinking mode (`enable_thinking=True`)
|
||||
|
||||
This is the mode for reasoning, math, coding, logic — anything that benefits from step-by-step generation.
|
||||
|
||||
**Use these generation settings:**
|
||||
|
||||
```
|
||||
Temperature: 0.6
|
||||
TopP: 0.95
|
||||
TopK: 20
|
||||
Max tokens: 32,768
|
||||
Do not use greedy decoding
|
||||
```
|
||||
|
||||
<Aside type="note">
|
||||
**Why it matters:**
|
||||
Thinking mode is powerful, but greedy decoding kills its output. It'll repeat or get stuck. Stick to sampling, and give it enough token space to finish its thoughts.
|
||||
</Aside>
|
||||
|
||||
|
||||
## Quick summary
|
||||
|
||||
<img src={qwen3SettingsJanAi.src} alt="Qwen3 settings" />
|
||||
|
||||
### Non-thinking mode (`enable_thinking=False`)
|
||||
|
||||
This is for fast, general-purpose replies. Instruction following, chat, creative writing — no `<think>` block, no extra steps.
|
||||
|
||||
**Use these settings:**
|
||||
|
||||
```makefile
|
||||
Temperature: 0.7
|
||||
TopP: 0.8
|
||||
TopK: 20
|
||||
```
|
||||
|
||||
<Aside type="note">
|
||||
Non-thinking mode is faster and more efficient, so the default token length is usually enough. If you're not doing long-form tasks, don't worry about it.
|
||||
</Aside>
|
||||
|
||||
|
||||
|
||||
## Soft vs. hard switch
|
||||
|
||||
You can toggle thinking dynamically in the prompt using:
|
||||
|
||||
```
|
||||
/think # turns thinking ON
|
||||
/no_think # turns it OFF
|
||||
```
|
||||
|
||||
This works only if `enable_thinking=True` is set in the code. If you set it to False, the soft switch won't do anything.
|
||||
|
||||
|
||||
|
||||
### What most people miss
|
||||
|
||||
- **Don't log the `think` block in chat history.** Qwen recommends keeping only the final answer. Otherwise, the next reply gets bloated and off-topic.
|
||||
- **Greedy decoding is a trap.** It's tempting to use for consistency, but Qwen3's output gets worse - and sometimes broken - without sampling.
|
||||
- **YaRN isn't always needed.** The model supports up to 32k context by default. Use YaRN only if you regularly go beyond that.
|
||||
|
||||
---
|
||||
|
||||
## Running Qwen3 locally with Jan
|
||||
|
||||
The easiest way to run Qwen3-30B-A3B locally is through Jan.
|
||||
|
||||
1. Download and install [Jan](https://jan.ai)
|
||||
2. Open Jan and navigate to Jan Hub
|
||||
3. Find `Qwen3` and `Qwen3-30B-A3B` in the model list
|
||||
4. Click "Download" to get the model
|
||||
|
||||
### Qwen3 in Jan Hub
|
||||
|
||||
You can easily find Qwen3 models in Jan Hub:
|
||||
|
||||
<img src={qwen3InJanHub.src} alt="Qwen3 in Jan Hub" />
|
||||
|
||||
Once downloaded, Jan handles all the technical setup, so you can focus on using the model rather than configuring it. The settings we covered in this guide are automatically applied when you use Qwen3 through Jan.
|
||||
|
||||
### How to customize Qwen3-30B-A3B settings in Jan
|
||||
|
||||
You can also customize these settings anytime by opening the right panel in Jan and adjusting the parameters to match your needs.
|
||||
|
||||
<img src={qwen3SettingsInJan.src} alt="Qwen3 settings in Jan app" />
|
||||
|
||||
## Bottom Line
|
||||
|
||||
If you're running Qwen3-30B-A3B locally, treat it like two models in one. Flip the thinking mode based on the task, adjust the generation settings accordingly, and let it work how it was meant to.
|
||||
|
||||
## Need help?
|
||||
|
||||
<Aside type="note">
|
||||
[Join our Discord community](https://discord.gg/Exe46xPMbK) for support and tips on using Jan as your offline ChatGPT alternative.
|
||||
</Aside>
|
||||
@ -1,134 +0,0 @@
|
||||
---
|
||||
title: "RAG is not enough: Lessons from Beating GPT-3.5 on Specialized Tasks with Mistral 7B"
|
||||
description: We present a straightforward approach to customizing small, open-source models using fine-tuning and RAG that outperforms GPT-3.5 for specialized use cases.
|
||||
tags: RAG, opensource chatgpt alternative, outperform ChatGPT, Mistral
|
||||
date: 2024-03-25
|
||||
unlisted: true
|
||||
categories: research
|
||||
---
|
||||
|
||||
import CTABlog from '@/components/Blog/CTABlog.astro';
|
||||
|
||||
|
||||
# RAG is not enough: Lessons from Beating GPT-3.5 on Specialized Tasks with Mistral 7B
|
||||
|
||||
## Abstract
|
||||
|
||||
We present a straightforward approach to customizing small, open-source models using fine-tuning and RAG that outperforms GPT-3.5 for specialized use cases. With it, we achieved superior Q&A results of [technical documentation](https://nitro.jan.ai/docs) for a small codebase [codebase](https://github.com/menloresearch/nitro).
|
||||
|
||||
In short, (1) extending a general foundation model like [Mistral](https://huggingface.co/mistralai/Mistral-7B-v0.1) with strong math and coding, and (2) training it over a high-quality, synthetic dataset generated from the intended corpus, and (3) adding RAG capabilities, can lead to significant accuracy improvements.
|
||||
|
||||
Problems still arise with catastrophic forgetting in general tasks, commonly observed during specialized domain fine-tuning. In our case, this is likely exacerbated by our lack of access to Mistral’s original training dataset and various compression techniques used in our approach to keep the model small.
|
||||
|
||||
## Selecting a strong foundation model
|
||||
|
||||
[Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) outshines both [Meta's Llama-2 7B](https://huggingface.co/meta-llama/Llama-2-7b) and [Google's Gemma 7B](https://huggingface.co/google/gemma-7b) in key benchmarks, making it our choice for a base model. Starting with a strong foundation like Mistral allowed us to achieve greater accuracy in our specialized adaptations.
|
||||
|
||||
|
||||
|
||||
*Figure 1. Mistral 7B excels in benchmarks, ranking among the top foundational models.*
|
||||
|
||||
*Note: We are not sponsored by the Mistral team, though lots of folks like to run Mistral locally using [Jan](https://jan.ai/)., our desktop client.*
|
||||
|
||||
## Cost effectively improving the base model
|
||||
|
||||
Our technical use case required excellent math capabilities, an area where Mistral can underperform. Thus, we tested a number of Mistral model variants, from foundation models to finetunes to model merges, to find a stronger base model before proceeding to finetuning.
|
||||
|
||||
|
||||
|
||||
*Figure 2: The merged model, Stealth, doubles the mathematical capabilities of its foundational model while retaining the performance in other tasks.*
|
||||
|
||||
We found merging models is quick and cost-effective, enabling fast adjustments based on the result of each iteration.
|
||||
|
||||
We ended up with [Stealth 7B v1.1](https://huggingface.co/jan-hq/stealth-v1.1), a [SLERP](https://github.com/Digitous/LLM-SLERP-Merge) merge of Mistral with the following:
|
||||
|
||||
- [WizardMath](https://huggingface.co/WizardLM/WizardMath-7B-V1.1) for its math capabilities.
|
||||
- [WizardCoder](https://huggingface.co/WizardLM/WizardCoder-Python-7B-V1.0) for its coding capabilities.
|
||||
- Our own [Trinity](https://huggingface.co/jan-hq/trinity-v1.2) model for its versatility across general tasks.
|
||||
|
||||
This particular combination yielded the best tradeoff across mathematical & technical reasoning while retaining the most pre-merge performance on general tasks.
|
||||
|
||||
## **DPO finetuning**
|
||||
|
||||
Merging different LLMs can lead to a mixed answering style because each model was originally trained on different types of data.
|
||||
|
||||
Thus, we applied Direct Preference Optimization ([DPO](https://arxiv.org/abs/2305.18290)) using the [Intel's Orca DPO pairs](https://huggingface.co/datasets/Intel/orca_dpo_pairs) dataset, chosen for its helpful answering style in general, math and coding concentration.
|
||||
|
||||
This approach produced a final model - [Stealth 7B v1.2](https://huggingface.co/jan-hq/stealth-v1.2), aligned to our technical preferences and demonstrating minimal loss.
|
||||
|
||||
## **Using our own technical documentation**
|
||||
|
||||
With the base model ready, we started on our specific use case.
|
||||
|
||||
Jan is an open-source project enjoying strong growth, but at one point we began receiving a new support ticket every minute, which quickly overwhelmed our bootstrapped resources.
|
||||
|
||||
So, we directed our efforts toward training a model to answer user questions based on existing technical documentation.
|
||||
|
||||
Specifically, we trained it on on the [Nitro documentation](https://nitro.jan.ai/docs). For context, Nitro is the default inference engine for Jan. It’s a enterprise-ready server implementation of LlamaCPP, written in C++, with multimodal, queues, and other production-level server capabilities.
|
||||
|
||||
It made an interesting corpus because it was rife with post-2023 technical jargon, edge cases, and poor informational layout.
|
||||
|
||||
## Generating training data
|
||||
|
||||
The first step was to transform Nitro’s unstructured format into a synthetic Q&A dataset designed for [instruction tuning](https://arxiv.org/pdf/2109.01652.pdf).
|
||||
|
||||
The text was split into chunks of 300-token segments with 30-token overlaps. This helped to avoid a [lost-in-the-middle](https://arxiv.org/abs/2307.03172) problem where LLM can’t use context efficiently to answer given questions.
|
||||
|
||||
The chunks were then given to GPT-4 with 8k context length to generate 3800 Q&A pairs. The [training dataset](https://huggingface.co/datasets/jan-hq/nitro_binarized_v2) is available on HuggingFace.
|
||||
|
||||
## **Training**
|
||||
|
||||
Training was done with supervised finetuning (SFT) from the [Hugging Face's alignment-handbook](https://github.com/huggingface/alignment-handbook), per [Huggingface's Zephyr Beta](https://github.com/huggingface/alignment-handbook/tree/main/recipes/zephyr-7b-beta) guidelines.
|
||||
|
||||
We used consumer-grade, dual Nvidia RTX 4090s for the training. The end-to-end training took 18 minutes. We found optimal hyperparameters in LoRA for this specific task to be `r = 256` and `alpha = 512`.
|
||||
|
||||
This final model can be found [here on Huggingface](https://huggingface.co/jan-hq/nitro-v1.2-e3).
|
||||
|
||||
|
||||
|
||||
*Figure 3. Using the new finetuned model in [Jan](https://jan.ai/)*
|
||||
|
||||
## Improving results with RAG
|
||||
|
||||
As an additional step, we also added [Retrieval Augmented Generation (RAG)](https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/) as an experiment parameter.
|
||||
|
||||
A simple RAG setup was done using **[Llamaindex](https://www.llamaindex.ai/)** and the **[bge-en-base-v1.5 embedding](https://huggingface.co/BAAI/bge-base-en-v1.5)** model for efficient documentation retrieval and question-answering. You can find the RAG implementation [here](https://github.com/menloresearch/open-foundry/blob/main/rag-is-not-enough/rag/nitro_rag.ipynb).
|
||||
|
||||
## Benchmarking the Results
|
||||
|
||||
We curated a new set of [50 multiple-choice questions](https://github.com/menloresearch/open-foundry/blob/main/rag-is-not-enough/rag/mcq_nitro.csv) (MCQ) based on the Nitro docs. The questions had varying levels of difficulty and had trick components that challenged the model's ability to discern misleading information.
|
||||
|
||||
|
||||
|
||||
*Figure 4. Comparison between finetuned model and OpenAI's GPT*
|
||||
|
||||
**Results**
|
||||
|
||||
- GPT-3.5 with RAG: 56.7%
|
||||
- GPT-4 with RAG: 64.3%
|
||||
- Merged 7B Model ([Stealth 7B](https://huggingface.co/jan-hq/stealth-v1.3)) with RAG: 47.7%
|
||||
- Finetuned 7B Model (Nitro 7B) with RAG: 57.8%
|
||||
|
||||
This indicates that with task-specific training, we can improve an open-source, Small Language Model to the level of GPT-3.5 on domain knowledge.
|
||||
|
||||
Notably, the finetuned + RAG approach also demonstrated more consistency across benchmarking, as indicated by its lower standard deviation.
|
||||
|
||||
## Conclusion
|
||||
|
||||
We conclude that this combination of model merging + finetuning + RAG yields promise. This finding is relevant for teams and individuals that need specialized, technical small language models that need to run in resource-constrained or highly secured environments, where GPT may not be an option.
|
||||
|
||||
Anecdotally, we’ve had some success using this model in practice to onboard new team members to the Nitro codebase.
|
||||
|
||||
A full research report with more statistics can be found [here](https://github.com/menloresearch/open-foundry/blob/main/rag-is-not-enough/README.md).
|
||||
|
||||
# References
|
||||
|
||||
- [Catastrophic forgetting](https://arxiv.org/abs/2308.08747)
|
||||
- [Math specialization](https://arxiv.org/abs/2308.09583)
|
||||
- [Code specialization](https://arxiv.org/abs/2306.08568)
|
||||
- [Search specialization](https://github.com/SciPhi-AI/agent-search)
|
||||
- [Evol Instruct](https://github.com/nlpxucan/WizardLM)
|
||||
- [Lost in the middle](https://arxiv.org/abs/2307.03172)
|
||||
- [Instruction tuning](https://arxiv.org/pdf/2109.01652.pdf)
|
||||
|
||||
<CTABlog />
|
||||
@ -1,228 +0,0 @@
|
||||
---
|
||||
title: "How to run AI models locally as a beginner?"
|
||||
description: "A straightforward guide to running AI models locally on your computer, regardless of your background."
|
||||
tags: AI, local models, Jan, GGUF, privacy, local AI
|
||||
categories: guides
|
||||
date: 2025-01-31
|
||||
ogImage: assets/run-ai-locally-with-jan.jpg
|
||||
twitter:
|
||||
card: summary_large_image
|
||||
site: "@jandotai"
|
||||
title: "How to run AI models locally as a beginner?"
|
||||
description: "Learn how to run AI models locally on your computer for enhanced privacy and control. Perfect for beginners!"
|
||||
image: assets/run-ai-locally-with-jan.jpg
|
||||
---
|
||||
|
||||
import CTABlog from '@/components/Blog/CTABlog.astro';
|
||||
|
||||
import { Aside } from '@astrojs/starlight/components'
|
||||
|
||||
|
||||
# How to run AI models locally as a beginner?
|
||||
|
||||
Most people think running AI models locally is complicated. It's not. Anyone can run powerful AI models like DeepSeek, Llama, and Mistral on their own computer. This guide will show you how, even if you've never written a line of code.
|
||||
|
||||
## Quick steps:
|
||||
### 1. Download [Jan](https://jan.ai)
|
||||
|
||||
|
||||
*Download Jan from [jan.ai](https://jan.ai) - it's free and open source.*
|
||||
|
||||
### 2. Choose a model that fits your hardware
|
||||
|
||||
|
||||
*Jan helps you pick the right AI model for your computer.*
|
||||
|
||||
### 3. Start using AI locally
|
||||
|
||||
That's all to run your first AI model locally!
|
||||
|
||||
|
||||
*Jan's easy-to-use chat interface after installation.*
|
||||
|
||||
Keep reading to learn key terms of local AI and the things you should know before running AI models locally.
|
||||
|
||||
## How Local AI Works
|
||||
|
||||
Before diving into the details, let's understand how AI runs on your computer:
|
||||
|
||||
<Aside type="tip">
|
||||
**Why do we need special tools for local AI?**
|
||||
Think of AI models like compressed files - they need to be "unpacked" to work on your computer. Tools like llama.cpp do this job:
|
||||
- They make AI models run efficiently on regular computers
|
||||
- Convert complex AI math into something your computer understands
|
||||
- Help run large AI models even with limited resources
|
||||
</Aside>
|
||||
|
||||
|
||||
*llama.cpp helps millions of people run AI locally on their computers.*
|
||||
|
||||
<Aside type="tip">
|
||||
**What is GGUF and why do we need it?**
|
||||
|
||||
Original AI models are huge and complex - like trying to read a book in a language your computer doesn't understand. Here's where GGUF comes in:
|
||||
|
||||
1. **Problem it solves:**
|
||||
- Original AI models are too big (100s of GB)
|
||||
- They're designed for specialized AI computers
|
||||
- They use too much memory
|
||||
|
||||
2. **How GGUF helps:**
|
||||
- Converts models to a smaller size
|
||||
- Makes them work on regular computers
|
||||
- Keeps the AI smart while using less memory
|
||||
|
||||
When browsing models, you'll see "GGUF" in the name (like "DeepSeek-R1-GGUF"). Don't worry about finding them - Jan automatically shows you the right GGUF versions for your computer.
|
||||
</Aside>
|
||||
|
||||
## Understanding AI Models
|
||||
|
||||
Think of AI models like apps on your computer - some are light and quick to use, while others are bigger but can do more things. When you're choosing an AI model to run on your computer, you'll see names like "Llama-3-8B" or "Mistral-7B". Let's break down what this means in simple terms.
|
||||
|
||||
<Aside type="tip">
|
||||
The "B" in model names (like 7B) stands for "billion" - it's just telling you the size of the AI model. Just like how some apps take up more space on your computer, bigger AI models need more space on your computer.
|
||||
|
||||
- Smaller models (1-7B): Work great on most computers
|
||||
- Bigger models (13B+): Need more powerful computers but can do more complex tasks
|
||||
</Aside>
|
||||
|
||||
|
||||
*Jan Hub makes it easy to understand different model sizes and versions*
|
||||
|
||||
**Good news:** Jan helps you pick the right model size for your computer automatically! You don't need to worry about the technical details - just choose a model that matches what Jan recommends for your computer.
|
||||
|
||||
## What You Can Do with Local AI
|
||||
|
||||
<Aside type="note">
|
||||
Running AI locally gives you:
|
||||
- Complete privacy - your data stays on your computer
|
||||
- No internet needed - works offline
|
||||
- Full control - you decide what models to use
|
||||
- Free to use - no subscription fees
|
||||
</Aside>
|
||||
|
||||
## Hardware Requirements
|
||||
|
||||
Before downloading an AI model, consider checking if your computer can run it. Here's a basic guide:
|
||||
|
||||
**The basics your computer needs:**
|
||||
- A decent processor (CPU) - most computers from the last 5 years will work fine
|
||||
- At least 8GB of RAM - 16GB or more is better
|
||||
- Some free storage space - at least 5GB recommended
|
||||
|
||||
### What Models Can Your Computer Run?
|
||||
|
||||
| | | |
|
||||
|---|---|---|
|
||||
| Regular Laptop | 3B-7B models | Good for chatting and writing. Like having a helpful assistant |
|
||||
| Gaming Laptop | 7B-13B models | More capable. Better at complex tasks like coding and analysis |
|
||||
| Powerful Desktop | 13B+ models | Better performance. Great for professional work and advanced tasks |
|
||||
|
||||
<Aside type="note">
|
||||
**Not Sure About Your Computer?**
|
||||
Start with a smaller model (3B-7B) - Jan will help you choose one that works well on your system.
|
||||
</Aside>
|
||||
|
||||
## Getting Started with Models
|
||||
|
||||
### Model Versions
|
||||
|
||||
When browsing models in Jan, you'll see terms like "Q4", "Q6", or "Q8". Here's what that means in simple terms:
|
||||
|
||||
<Aside type="tip">
|
||||
These are different versions of the same AI model, just packaged differently to work better on different computers:
|
||||
|
||||
- **Q4 versions**: Like a "lite" version of an app - runs fast and works on most computers
|
||||
- **Q6 versions**: The "standard" version - good balance of speed and quality
|
||||
- **Q8 versions**: The "premium" version - highest quality but needs a more powerful computer
|
||||
</Aside>
|
||||
|
||||
**Pro tip**: Start with Q4 versions - they work great for most people and run smoothly on regular computers!
|
||||
|
||||
### Getting Models from Hugging Face
|
||||
|
||||
You'll often see links to "Hugging Face" when downloading AI models. Think of Hugging Face as the "GitHub for AI" - it's where the AI community shares their models. Jan makes it super easy to use:
|
||||
|
||||
1. Jan has a built-in connection to Hugging Face
|
||||
2. You can download models right from Jan's interface
|
||||
3. No need to visit the Hugging Face website unless you want to explore more options
|
||||
|
||||
## Setting up your local AI
|
||||
|
||||
### Getting Models from Hugging Face
|
||||
|
||||
You'll often see links to "Hugging Face" when downloading AI models. Think of Hugging Face as the "GitHub for AI" - it's where the AI community shares their models. This sounds technical, but Jan makes it super easy to use:
|
||||
|
||||
1. Jan has a built-in connection to Hugging Face
|
||||
2. You can download models right from Jan's interface
|
||||
3. No need to visit the Hugging Face website unless you want to explore more options
|
||||
|
||||
<Aside type="tip">
|
||||
**What powers local AI?**
|
||||
Jan uses [llama.cpp](https://github.com/ggerganov/llama.cpp), an inference that makes AI models run efficiently on regular computers. It's like a translator that helps AI models speak your computer's language, making them run faster and use less memory.
|
||||
</Aside>
|
||||
|
||||
### 1. Get Started
|
||||
Download Jan from [jan.ai](https://jan.ai) - it sets everything up for you.
|
||||
|
||||
### 2. Get an AI Model
|
||||
|
||||
You can get models two ways:
|
||||
|
||||
#### 1. Use Jan Hub (Recommended):
|
||||
- Click "Download Model" in Jan
|
||||
- Pick a recommended model
|
||||
- Choose one that fits your computer
|
||||
|
||||
|
||||
*Use Jan Hub to download AI models*
|
||||
|
||||
#### 2. Use Hugging Face:
|
||||
|
||||
<Aside type="caution">
|
||||
Important: Only GGUF models will work with Jan. Make sure to use models that have "GGUF" in their name.
|
||||
</Aside>
|
||||
|
||||
##### Step 1: Get the model link
|
||||
Find and copy a GGUF model link from [Hugging Face](https://huggingface.co)
|
||||
|
||||
|
||||
*Look for models with "GGUF" in their name*
|
||||
|
||||
##### Step 2: Open Jan
|
||||
Launch Jan and go to the Models tab
|
||||
|
||||
|
||||
*Navigate to the Models section in Jan*
|
||||
|
||||
##### Step 3: Add the model
|
||||
Paste your Hugging Face link into Jan
|
||||
|
||||
|
||||
*Paste your GGUF model link here*
|
||||
|
||||
##### Step 4: Download
|
||||
Select your quantization and start the download
|
||||
|
||||
|
||||
*Choose your preferred model size and download*
|
||||
|
||||
### Common Questions
|
||||
|
||||
**"My computer doesn't have a graphics card - can I still use AI?"**
|
||||
|
||||
Yes! It will run slower but still work. Start with 7B models.
|
||||
|
||||
**"Which model should I start with?"**
|
||||
|
||||
Try a 7B model first - it's the best balance of smart and fast.
|
||||
|
||||
**"Will it slow down my computer?"**
|
||||
|
||||
Only while you're using the AI. Close other big programs for better speed.
|
||||
|
||||
## Need help?
|
||||
<Aside type="note">
|
||||
[Join our Discord community](https://discord.gg/Exe46xPMbK) for support.
|
||||
</Aside>
|
||||
<CTABlog />
|
||||
@ -1,17 +0,0 @@
|
||||
---
|
||||
title: "Faster inference across: Mac, Windows, Linux, and GPUs"
|
||||
version: 0.4.3
|
||||
description: ""
|
||||
date: 2023-12-21
|
||||
ogImage: "/assets/images/changelog/Jan_v0.4.3.gif"
|
||||
---
|
||||
|
||||
import ChangelogHeader from '@/components/Changelog/ChangelogHeader.astro'
|
||||
|
||||
<ChangelogHeader title= "Faster inference across: Mac, Windows, Linux, and GPUs" date= "2023-12-21" ogImage= "/assets/images/changelog/Jan_v0.4.3.gif" />
|
||||
|
||||
### Highlights 🎉
|
||||
|
||||
- Custom models: `Trinity`, `Pandora` (great for general use).
|
||||
- Faster inference across: Mac, Windows, Linux, and GPUs.
|
||||
- Connect to remote OpenAI models like GPT4 via API key.
|
||||
@ -1,20 +0,0 @@
|
||||
---
|
||||
title: "Thread settings options in the right panel"
|
||||
version: 0.4.4
|
||||
description: ""
|
||||
date: 2024-01-16
|
||||
ogImage: ""
|
||||
---
|
||||
|
||||
import ChangelogHeader from '@/components/Changelog/ChangelogHeader.astro'
|
||||
|
||||
<ChangelogHeader title= "Thread settings options in the right panel" date= "2024-01-16" ogImage= "" />
|
||||
|
||||
### Highlights 🎉
|
||||
|
||||
- You can now see whether the model is compatible with running on your device.
|
||||
- You can switch model mid-threads.
|
||||
- More thread settings options in the right panel.
|
||||
- CI automation, anti-virus checks.
|
||||
- Social media access to Jan's Discord & Github from the app for further user support.
|
||||
- Fixed major bugs, more stability.
|
||||
@ -1,23 +0,0 @@
|
||||
---
|
||||
title: "Local API server"
|
||||
version: 0.4.5
|
||||
description: ""
|
||||
date: 2024-01-29
|
||||
ogImage: "/assets/images/changelog/Jan_v0.4.5.gif"
|
||||
---
|
||||
|
||||
import ChangelogHeader from '@/components/Changelog/ChangelogHeader.astro'
|
||||
|
||||
<ChangelogHeader title= "Local API server" date= "2024-01-29" ogImage= "/assets/images/changelog/Jan_v0.4.5.gif" />
|
||||
|
||||
### Highlights 🎉
|
||||
|
||||
- Local API Server: Experience Jan's dashboard for the local API server to make your data processing smoother and more efficient.
|
||||
- HTTP Proxy Support: Now, downloading and connecting are seamless, just like browsing Hugging Face in your browser.
|
||||
- Updated Settings Page: Find what you need faster! We've updated the settings page.
|
||||
|
||||
### Fixes 💫
|
||||
|
||||
- Auto Update: Enjoy smoother updates. We've fixed the glitches.
|
||||
- Swagger API Page: Full documentation, no more blanks.
|
||||
- GPU for Models: Your imported models now fully leverage GPU power.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user