Index: conf/files =================================================================== RCS file: /home/ncvs/src/sys/conf/files,v retrieving revision 1.340.2.51 diff -u -r1.340.2.51 files --- conf/files 2001/03/05 05:33:20 1.340.2.51 +++ conf/files 2001/07/10 15:14:45 @@ -896,6 +896,7 @@ nfs/nfs_syscalls.c optional nfs nfs/nfs_vfsops.c optional nfs nfs/nfs_vnops.c optional nfs +kern/embuf.c optional nfs nfs/bootp_subr.c optional bootp nfs/krpc_subr.c optional bootp nwfs/nwfs_io.c optional nwfs Index: conf/options =================================================================== RCS file: /home/ncvs/src/sys/conf/options,v retrieving revision 1.191.2.22 diff -u -r1.191.2.22 options --- conf/options 2001/04/05 17:23:43 1.191.2.22 +++ conf/options 2001/05/17 18:47:09 @@ -342,6 +342,7 @@ DEBUG_VFS_LOCKS opt_global.h DIAGNOSTIC opt_global.h ENABLE_VFS_IOOPT opt_global.h +ZERO_COPY_SOCKETS opt_global.h INVARIANT_SUPPORT opt_global.h INVARIANTS opt_global.h SIMPLELOCK_DEBUG opt_global.h Index: kern/kern_subr.c =================================================================== RCS file: /home/ncvs/src/sys/kern/kern_subr.c,v retrieving revision 1.31 diff -u -r1.31 kern_subr.c --- kern/kern_subr.c 1999/10/29 18:08:51 1.31 +++ kern/kern_subr.c 2001/05/17 20:02:39 @@ -54,6 +54,81 @@ static void uio_yield __P((void)); +#if defined(ZERO_COPY_SOCKETS) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int vm_pgmoveco __P((vm_map_t, vm_object_t, vm_offset_t, + vm_offset_t)); +static int userspaceco __P((caddr_t, u_int, struct uio *, + struct vm_object *, int)); +extern int zcs_enable; + +static int +vm_pgmoveco(mapa, srcobj, kaddr, uaddr) + vm_map_t mapa; + vm_object_t srcobj; + vm_offset_t kaddr, uaddr; +{ + vm_map_t map = mapa; + vm_page_t kern_pg, user_pg; + vm_object_t uobject; + vm_map_entry_t entry; + vm_pindex_t upindex, kpindex; + vm_prot_t prot; + boolean_t wired; + + /* + * First lookup the kernel page. + */ + kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr)); + + if ((vm_map_lookup(&map, uaddr, + VM_PROT_READ, &entry, &uobject, + &upindex, &prot, &wired)) != KERN_SUCCESS) { + return(EFAULT); + } + if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { + vm_page_sleep_busy(user_pg, 1, "vm_pgmoveco"); + pmap_remove(map->pmap, uaddr, uaddr+PAGE_SIZE); + vm_page_busy(user_pg); + vm_page_free(user_pg); + } + + if (kern_pg->busy || ((kern_pg->queue - kern_pg->pc) == PQ_FREE) || + (kern_pg->hold_count != 0)|| (kern_pg->flags & PG_BUSY)) { + printf("vm_pgmoveco: pindex(%lu), busy(%d), PG_BUSY(%d), " + "hold(%d) paddr(0x%lx)\n", (u_long)kern_pg->pindex, + kern_pg->busy, (kern_pg->flags & PG_BUSY) ? 1 : 0, + kern_pg->hold_count, (u_long)kern_pg->phys_addr); + if ((kern_pg->queue - kern_pg->pc) == PQ_FREE) + panic("vm_pgmoveco: renaming free page"); + else + panic("vm_pgmoveco: renaming busy page"); + } + kpindex = kern_pg->pindex; + vm_page_busy(kern_pg); + vm_page_rename(kern_pg, uobject, upindex); + vm_page_flag_clear(kern_pg, PG_BUSY); + kern_pg->valid = VM_PAGE_BITS_ALL; + + vm_map_lookup_done(map, entry); + return(KERN_SUCCESS); +} +#endif /* ZERO_COPY_SOCKETS */ + int uiomove(cp, n, uio) register caddr_t cp; @@ -121,12 +196,96 @@ return (error); } +static int +userspaceco(cp, cnt, uio, obj, disposable) + caddr_t cp; + u_int cnt; + struct uio *uio; + struct vm_object *obj; + int disposable; +{ + struct iovec *iov; + int error; + + iov = uio->uio_iov; + +#if defined(ZERO_COPY_SOCKETS) && defined(ENABLE_VFS_IOOPT) + + if (uio->uio_rw == UIO_READ) { + if ((obj != NULL) + && ((cnt & PAGE_MASK) == 0) + && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) + && ((uio->uio_offset & PAGE_MASK) == 0) + && ((((intptr_t) cp) & PAGE_MASK) == 0)) { + if (zcs_enable + && (obj->type == OBJT_DEFAULT) + && (disposable != 0)) { + /* SOCKET: use page-trading */ + /* + * We only want to call vm_pgmoveco() on + * disposeable pages, since it gives the + * kernel page to the userland process. + */ + error = vm_pgmoveco(&curproc->p_vmspace->vm_map, + obj, (vm_offset_t)cp, + (vm_offset_t)iov->iov_base); + + /* + * If we get an error back, attempt + * to use copyout() instead. The + * disposable page should be freed + * automatically if we weren't able to move + * it into userland. + */ + if (error != 0) + error = copyout(cp, iov->iov_base, cnt); + } else { + /* something else, use vm_uiomove */ + if (vfs_ioopt) + error = vm_uiomove(&curproc->p_vmspace->vm_map, + obj, uio->uio_offset, cnt, + (vm_offset_t) iov->iov_base, + NULL); + else + error = copyout(cp, iov->iov_base, cnt); + } + } else { + error = copyout(cp, iov->iov_base, cnt); + } + } else { + error = copyin(iov->iov_base, cp, cnt); + } +#else /*ZERO_COPY_SOCKETS && ENABLE_VFS_IOOPT */ + if (uio->uio_rw == UIO_READ) { +#ifdef ENABLE_VFS_IOOPT + if ((vfs_ioopt != 0) + && ((cnt & PAGE_MASK) == 0) + && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) + && ((uio->uio_offset & PAGE_MASK) == 0) + && ((((intptr_t) cp) & PAGE_MASK) == 0)) { + error = vm_uiomove(&curproc->p_vmspace->vm_map, obj, + uio->uio_offset, cnt, + (vm_offset_t) iov->iov_base, NULL); + } else +#endif /* ENABLE_VFS_IOOPT */ + { + error = copyout(cp, iov->iov_base, cnt); + } + } else { + error = copyin(iov->iov_base, cp, cnt); + } +#endif /*ZERO_COPY_SOCKETS && ENABLE_VFS_IOOPT*/ + + return(error); +} + int -uiomoveco(cp, n, uio, obj) +uiomoveco(cp, n, uio, obj, disposable) caddr_t cp; int n; struct uio *uio; struct vm_object *obj; + int disposable; { struct iovec *iov; u_int cnt; @@ -154,23 +313,9 @@ case UIO_USERISPACE: if (ticks - switchticks >= hogticks) uio_yield(); - if (uio->uio_rw == UIO_READ) { -#ifdef ENABLE_VFS_IOOPT - if (vfs_ioopt && ((cnt & PAGE_MASK) == 0) && - ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) && - ((uio->uio_offset & PAGE_MASK) == 0) && - ((((intptr_t) cp) & PAGE_MASK) == 0)) { - error = vm_uiomove(&curproc->p_vmspace->vm_map, obj, - uio->uio_offset, cnt, - (vm_offset_t) iov->iov_base, NULL); - } else -#endif - { - error = copyout(cp, iov->iov_base, cnt); - } - } else { - error = copyin(iov->iov_base, cp, cnt); - } + + error = userspaceco(cp, cnt, uio, obj, disposable); + if (error) return (error); break; @@ -427,4 +572,209 @@ p->p_stats->p_ru.ru_nivcsw++; mi_switch(); splx(s); +} + +/*- + * Copyright (c) 1997-2001, Duke University + * All rights reserved. + * + * Author: + * Andrew Gallatin + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgements: + * This product includes software developed by Duke University + * 4. The name of Duke University may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY DUKE UNIVERSITY ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DUKE UNIVERSITY BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITSOR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +/* + * This is a set of routines for enabling and disabling copy on write + * protection for data written into sockets. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +struct netsend_cow_stats { + int attempted; + int fail_not_mapped; + int fail_wired; + int fail_not_anon; + int fail_pmap_cow; + int fail_pg_error; + int fail_kva; + int free_post_exit; + int success; + int iodone; + int freed; +}; + +static struct netsend_cow_stats socow_stats = {0,0,0,0,0,0,0,0,0,0,0}; + +extern struct sf_buf *sf_bufs; +extern vm_offset_t sf_base; +#define dtosf(x) (&sf_bufs[((uintptr_t)(x) - (uintptr_t)sf_base) >> PAGE_SHIFT]) +void sf_buf_free(caddr_t addr, void *args); +struct sf_buf *sf_buf_alloc(void); +void sf_buf_ref(caddr_t addr, u_int size); +static void socow_iodone __P((caddr_t addr, u_int len)); + +static void +socow_iodone(addr, len) + caddr_t addr; + u_int len; +{ + int s; + struct sf_buf *sf; + + vm_offset_t paddr; + vm_page_t pp; + + sf = dtosf(addr); + if (sf->refcnt == 0) + panic("sf_buf_free: freeing free sf_buf"); + if(sf->refcnt > 1) + sf->refcnt--; + else { + paddr = vtophys((vm_offset_t)addr); + pp = PHYS_TO_VM_PAGE(paddr); + s = splvm(); + /* remove COW mapping */ + vm_page_cowclear(pp); + if (pp->object && pp->object->ref_count) + vm_object_deallocate(pp->object); + splx(s); + /* note that sf_buf_free() unwires the page for us*/ + sf_buf_free(addr, NULL); + } + socow_stats.iodone++; +} + +int +socow_setup(m0, uio) + struct mbuf *m0; + struct uio *uio; +{ + struct sf_buf *sf; + vm_page_t pp; + vm_offset_t pa; + struct iovec *iov; + struct vmspace *vmspace; + struct vm_map *map; + vm_offset_t uva; + int s; + + vmspace = curproc->p_vmspace;; + map = &vmspace->vm_map; + uva = (vm_offset_t) uio->uio_iov->iov_base; + + s = splvm(); + + /* + * verify page is mapped & not already wired for i/o + */ + socow_stats.attempted++; + pa=pmap_extract(map->pmap, uva); + if(!pa) { + socow_stats.fail_not_mapped++; + splx(s); + return(0); + } + pp = PHYS_TO_VM_PAGE(pa); + + sf = sf_buf_alloc(); + sf->m = pp; + pmap_qenter(sf->kva, &pp, 1); + + /* + * set up COW + */ + vm_page_cowsetup(pp); + + /* + * wire the page for I/O + */ + vm_page_wire(pp); + + /* + * prevent the process from exiting on us. + */ + vm_object_reference(pp->object); + + /* + * attach to mbuf + */ + m0->m_data = m0->m_ext.ext_buf = (caddr_t)sf->kva; + m0->m_len = PAGE_SIZE; + m0->m_ext.ext_size = PAGE_SIZE; + m0->m_ext.ext_free = socow_iodone; + m0->m_ext.ext_ref = sf_buf_ref; + m0->m_flags |= (M_EXT); + socow_stats.success++; + + + iov = uio->uio_iov; + iov->iov_base += PAGE_SIZE; + iov->iov_len -= PAGE_SIZE; + uio->uio_resid -= PAGE_SIZE; + uio->uio_offset += PAGE_SIZE; + if (iov->iov_len == 0) { + uio->uio_iov++; + uio->uio_iovcnt--; + } + + + splx(s); + return(1); } Index: kern/uipc_mbuf.c =================================================================== RCS file: /home/ncvs/src/sys/kern/uipc_mbuf.c,v retrieving revision 1.51.2.3 diff -u -r1.51.2.3 uipc_mbuf.c --- kern/uipc_mbuf.c 2000/08/25 23:23:32 1.51.2.3 +++ kern/uipc_mbuf.c 2001/05/17 18:56:38 @@ -1198,3 +1198,267 @@ } return; } + +/* + * Copyright (c) 1999-2001, Duke University + * All rights reserved. + * + * Author: + * Andrew Gallatin + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgements: + * This product includes software developed by Duke University + * 4. The name of Duke University may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY DUKE UNIVERSITY ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DUKE UNIVERSITY BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITSOR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* XXX KDM XXX */ +#define JUMBO_MAX_PAGES 8192 + +SLIST_HEAD(jumbo_kmap_head, jumbo_kmap) jumbo_kmap_free, jumbo_kmap_inuse; + struct jumbo_kmap { + vm_offset_t kva; + SLIST_ENTRY(jumbo_kmap) entries; /* Singly-linked List. */ + }; + + +struct vm_object *jumbo_vm_object; +unsigned long jumbo_vmuiomove_pgs_freed=0; +int jumbo_vm_wakeup_wanted=0; +vm_offset_t jumbo_basekva; +/* XXX KDM XXX */ + + +caddr_t +jumbo_phys_to_kva (vm_offset_t pa) +{ + vm_page_t pg = PHYS_TO_VM_PAGE(pa); + pg->flags &= ~PG_BUSY; + return (caddr_t)(ptoa(pg->pindex) + jumbo_basekva); +} + +int +jumbo_vm_init(void) +{ + int i; + struct jumbo_kmap *entry; + + if(jumbo_vm_object) + return(1); +/* allocate our object */ + jumbo_vm_object = vm_object_allocate(OBJT_DEFAULT, JUMBO_MAX_PAGES); + +/* grab some kernel virtual address space */ + SLIST_INIT(&jumbo_kmap_free); + SLIST_INIT(&jumbo_kmap_inuse); + jumbo_basekva = kmem_alloc_pageable(kernel_map, PAGE_SIZE * JUMBO_MAX_PAGES); + if(jumbo_basekva == 0) { + vm_object_deallocate(jumbo_vm_object); + jumbo_vm_object = NULL; + return 0; + } + for (i = 0; i < JUMBO_MAX_PAGES; i++){ + entry = malloc(sizeof(struct jumbo_kmap), M_TEMP, M_WAITOK); + if(!entry && !i) + panic("jumbo unable to allocated kvas"); + else if(!entry){ + printf("warning: ti allocated only %d kva\n",i); + return 1; + } + entry->kva = jumbo_basekva + (vm_offset_t)i * PAGE_SIZE; + SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries); + } + return 1; +} + +void +jumbo_freem(caddr_t addr, u_int len) +{ + register int s; + vm_page_t frame = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)addr)); + s = splimp(); + if((frame->object == jumbo_vm_object) && (frame->hold_count)){ + frame->hold_count--; + } else{ + jumbo_pg_free((vm_offset_t)addr); + } + splx(s); +} + +void +jumbo_ref(caddr_t addr, u_int len) +{ + vm_page_t frame = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)addr)); + frame->hold_count++; +} + +/* + * somewhat racy -- this is only safe if called when we "know" + * that no other thread can aquire a reference to our mbuf + */ + +int +jumbo_disposable(caddr_t addr) +{ + vm_page_t frame = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)addr)); + + if (frame->object != jumbo_vm_object) + return 0; + if (frame->hold_count) + return 0; + return 1; +} + +void +jumbo_pg_steal(vm_page_t pg) +{ + vm_offset_t addr = ptoa(pg->pindex) + jumbo_basekva; + struct jumbo_kmap *entry; + int s; + if(pg->object != jumbo_vm_object) + panic("stealing a non jumbo_vm_object page"); + vm_page_remove(pg); + s = splvm(); + pmap_qremove(addr,1); + entry = jumbo_kmap_inuse.slh_first; + entry->kva = addr; + SLIST_REMOVE_HEAD(&jumbo_kmap_inuse, entries); + SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries); + splx(s); + if(jumbo_vm_wakeup_wanted) + wakeup(jumbo_vm_object); +} + + +vm_page_t +jumbo_pg_alloc(void) +{ + vm_page_t pg = NULL; + vm_pindex_t pindex; + struct jumbo_kmap *entry; + int s = splvm(); + + entry = jumbo_kmap_free.slh_first; + if(entry != NULL){ + pindex = atop(entry->kva - jumbo_basekva); + pg = vm_page_alloc(jumbo_vm_object, pindex, VM_ALLOC_INTERRUPT); + if(pg != NULL){ + SLIST_REMOVE_HEAD(&jumbo_kmap_free, entries); + SLIST_INSERT_HEAD(&jumbo_kmap_inuse, entry, entries); + pmap_qenter(entry->kva, &pg, 1); + } + } + splx(s); + return(pg); +} + +void +jumbo_pg_free(vm_offset_t addr) +{ + int s; + struct jumbo_kmap *entry; + vm_offset_t paddr = pmap_kextract((vm_offset_t)addr); + vm_page_t pg = PHYS_TO_VM_PAGE(paddr); + + + if(pg->object != jumbo_vm_object){ + jumbo_vmuiomove_pgs_freed++; +/* if(vm_page_lookup(jumbo_vm_object, atop(addr - jumbo_basekva))) + panic("vm_page_rename didn't"); + printf("freeing uiomoved pg:\t pindex = %d, padd = 0x%lx\n", + atop(addr - jumbo_basekva), paddr); +*/ + } else { + vm_page_busy(pg); /* vm_page_free wants pages to be busy*/ + vm_page_free(pg); + } + s = splvm(); + pmap_qremove(addr,1); + entry = jumbo_kmap_inuse.slh_first; + entry->kva = addr; + SLIST_REMOVE_HEAD(&jumbo_kmap_inuse, entries); + SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries); + splx(s); + if(jumbo_vm_wakeup_wanted) + wakeup(jumbo_vm_object); +} + +#if 0 +/* + * deallocate jumbo buffers & vm object + * NOTE: There may still be some ext. mbufs floating around + * which point at our pages. We must wait until the + * jumbo_vm_object is empty before deallocating it. + */ + +static void +jumbo_free_jumbos(device_t dev) +{ + + int i, s; + struct jumbo_kmap *entry, *next_entry; + + s = splvm(); + + while(jumbo_vm_object->resident_page_count){ + device_printf(dev, "jumbo_free_jumbos: jumbo_vm_object has %d resident pages remaining, sleeping..\n", + jumbo_vm_object->resident_page_count); + jumbo_vm_wakeup_wanted = 1; + tsleep(jumbo_vm_object, PVM, "jumbo_free_jumbos", 0); + } + /* + * Free queue structures + */ + + next_entry = SLIST_FIRST(&jumbo_kmap_free); + entry = next_entry; + i = 0; + while(entry) { + next_entry = SLIST_NEXT(entry, entries); + free(entry, M_TEMP); + entry = next_entry; + i++; + } + if(i != JUMBO_MAX_PAGES) { + device_printf(dev, "DANGER, DANGER! freed only %d kmap_entries (out of %d)\n", + i, JUMBO_MAX_PAGES); + } + vm_object_deallocate(jumbo_vm_object); + jumbo_vm_object = NULL; + splx(s); +} +#endif Index: kern/uipc_socket.c =================================================================== RCS file: /home/ncvs/src/sys/kern/uipc_socket.c,v retrieving revision 1.68.2.15 diff -u -r1.68.2.15 uipc_socket.c --- kern/uipc_socket.c 2001/03/09 16:41:20 1.68.2.15 +++ kern/uipc_socket.c 2001/05/17 19:21:32 @@ -89,6 +89,26 @@ SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW, &somaxconn, 0, "Maximum pending socket connection queue size"); +#ifdef ZERO_COPY_SOCKETS +#include +#include +#include +#include +#include +#include + +int zcs_enable = 0; +struct so_zerocopy_stats{ + int size_ok; + int align_ok; + int found_ifp; +}; +struct so_zerocopy_stats so_zerocp_stats = {0,0,0}; + +SYSCTL_INT(_kern_ipc, 0, zcs_enable, CTLFLAG_RW, + &zcs_enable, 0, "enable zero copy sockets"); + +#endif /* * Socket operation routines. * These routines are called by the routines in @@ -460,6 +480,9 @@ register long space, len, resid; int clen = 0, error, s, dontroute, mlen; int atomic = sosendallatonce(so) || top; +#ifdef ZERO_COPY_SOCKETS + int cow_send; +#endif if (uio) resid = uio->uio_resid; @@ -548,6 +571,9 @@ if (flags & MSG_EOR) top->m_flags |= M_EOR; } else do { +#ifdef ZERO_COPY_SOCKETS + cow_send = 0; +#endif if (top == 0) { MGETHDR(m, M_WAIT, MT_DATA); if (m == NULL) { @@ -566,12 +592,30 @@ mlen = MLEN; } if (resid >= MINCLSIZE) { +#ifdef ZERO_COPY_SOCKETS + if (zcs_enable && resid >= PAGE_SIZE && + space >= PAGE_SIZE && + uio->uio_iov->iov_len>=PAGE_SIZE) { + so_zerocp_stats.size_ok++; + if(!((vm_offset_t) + uio->uio_iov->iov_base & PAGE_MASK)){ + so_zerocp_stats.align_ok++; + cow_send = socow_setup(m, uio); + } + } + if (!cow_send) { +#endif MCLGET(m, M_WAIT); if ((m->m_flags & M_EXT) == 0) goto nopages; mlen = MCLBYTES; len = min(min(mlen, resid), space); } else { +#ifdef ZERO_COPY_SOCKETS + len = PAGE_SIZE; + } + } else { +#endif nopages: len = min(min(mlen, resid), space); /* @@ -582,7 +626,12 @@ MH_ALIGN(m, len); } space -= len; +#ifdef ZERO_COPY_SOCKETS + if (cow_send) + error = 0; + else error = uiomove(mtod(m, caddr_t), (int)len, uio); +#endif resid = uio->uio_resid; m->m_len = len; *mp = m; @@ -693,6 +742,26 @@ if (error) goto bad; do { +#if defined(ZERO_COPY_SOCKETS) && defined(ENABLE_VFS_IOOPT) + if (zcs_enable) { + vm_page_t pg; + int disposable; + + if ((m->m_flags & M_EXT) + && jumbo_disposable(mtod(m, caddr_t))) + disposable = 1; + else + disposable = 0; + + pg = PHYS_TO_VM_PAGE(vtophys(mtod(m, caddr_t))); + if (uio->uio_offset == -1) + uio->uio_offset = IDX_TO_OFF(pg->pindex); + + error = uiomoveco(mtod(m, caddr_t), + min(uio->uio_resid, m->m_len), + uio, pg->object, disposable); + } else +#endif /*ZERO_COPY_SOCKETS && ENABLE_VFS_IOOPT*/ error = uiomove(mtod(m, caddr_t), (int) min(uio->uio_resid, m->m_len), uio); m = m_free(m); @@ -846,6 +915,26 @@ */ if (mp == 0) { splx(s); +#if defined(ZERO_COPY_SOCKETS) && defined(ENABLE_VFS_IOOPT) + if (zcs_enable) { + vm_page_t pg; + int disposable; + + if ((m->m_flags & M_EXT) + && jumbo_disposable(mtod(m, caddr_t) + moff)) + disposable = 1; + else + disposable = 0; + + pg = PHYS_TO_VM_PAGE(vtophys(mtod(m, caddr_t) + moff)); + if (uio->uio_offset == -1) + uio->uio_offset = IDX_TO_OFF(pg->pindex); + + error = uiomoveco(mtod(m, caddr_t) + moff, + min(uio->uio_resid, m->m_len), + uio, pg->object, disposable); + } else +#endif /*ZERO_COPY_SOCKETS && ENABLE_VFS_IOOPT*/ error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio); s = splnet(); if (error) Index: kern/uipc_syscalls.c =================================================================== RCS file: /home/ncvs/src/sys/kern/uipc_syscalls.c,v retrieving revision 1.65.2.7 diff -u -r1.65.2.7 uipc_syscalls.c --- kern/uipc_syscalls.c 2001/02/24 18:37:26 1.65.2.7 +++ kern/uipc_syscalls.c 2001/05/17 19:23:00 @@ -71,9 +71,9 @@ static void sf_buf_init(void *arg); SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL) -static struct sf_buf *sf_buf_alloc(void); -static void sf_buf_ref(caddr_t addr, u_int size); -static void sf_buf_free(caddr_t addr, u_int size); +struct sf_buf *sf_buf_alloc(void); +void sf_buf_ref(caddr_t addr, u_int size); +void sf_buf_free(caddr_t addr, u_int size); static int sendit __P((struct proc *p, int s, struct msghdr *mp, int flags)); static int recvit __P((struct proc *p, int s, struct msghdr *mp, @@ -86,9 +86,9 @@ int compat)); static SLIST_HEAD(, sf_buf) sf_freelist; -static vm_offset_t sf_base; -static struct sf_buf *sf_bufs; -static int sf_buf_alloc_want; +vm_offset_t sf_base; +struct sf_buf *sf_bufs; +int sf_buf_alloc_want; /* * System call interface to the socket abstraction. @@ -1436,7 +1436,7 @@ /* * Get an sf_buf from the freelist. Will block if none are available. */ -static struct sf_buf * +struct sf_buf * sf_buf_alloc() { struct sf_buf *sf; @@ -1454,7 +1454,7 @@ } #define dtosf(x) (&sf_bufs[((uintptr_t)(x) - (uintptr_t)sf_base) >> PAGE_SHIFT]) -static void +void sf_buf_ref(caddr_t addr, u_int size) { struct sf_buf *sf; @@ -1471,7 +1471,7 @@ * * Must be called at splimp. */ -static void +void sf_buf_free(caddr_t addr, u_int size) { struct sf_buf *sf; Index: modules/md/Makefile =================================================================== RCS file: /home/ncvs/src/sys/modules/md/Makefile,v retrieving revision 1.3 diff -u -r1.3 Makefile --- modules/md/Makefile 1999/11/28 18:52:57 1.3 +++ modules/md/Makefile 2001/05/18 17:53:41 @@ -1,8 +1,9 @@ # $FreeBSD: src/sys/modules/md/Makefile,v 1.3 1999/11/28 18:52:57 bde Exp $ -.PATH: ${.CURDIR}/../../dev/md +.PATH: ${.CURDIR}/../../dev/md + KMOD= md -SRCS= md.c opt_mfs.h opt_md.h +SRCS= md.c opt_mfs.h opt_md.h vnode_if.h NOMAN= .include Index: modules/mii/Makefile =================================================================== RCS file: /home/ncvs/src/sys/modules/mii/Makefile,v retrieving revision 1.11.2.2 diff -u -r1.11.2.2 Makefile --- modules/mii/Makefile 2000/10/03 18:46:55 1.11.2.2 +++ modules/mii/Makefile 2001/07/02 19:07:31 @@ -6,5 +6,6 @@ SRCS += miibus_if.h device_if.h miibus_if.c exphy.c nsphy.c SRCS += mlphy.c tlphy.c rlphy.c amphy.c dcphy.c pnphy.c SRCS += pnaphy.c brgphy.c xmphy.c +SRCS += nsgphy.c .include Index: modules/ti/Makefile =================================================================== RCS file: /home/ncvs/src/sys/modules/ti/Makefile,v retrieving revision 1.8 diff -u -r1.8 Makefile --- modules/ti/Makefile 2000/01/28 11:26:38 1.8 +++ modules/ti/Makefile 2001/05/14 15:34:35 @@ -2,7 +2,7 @@ .PATH: ${.CURDIR}/../../pci KMOD = if_ti -SRCS = if_ti.c opt_bdg.h vlan.h device_if.h bus_if.h pci_if.h +SRCS = if_ti.c opt_bdg.h vlan.h device_if.h bus_if.h pci_if.h vnode_if.h CLEANFILES = vlan.h vlan.h: Index: netinet/ip_output.c =================================================================== RCS file: /home/ncvs/src/sys/netinet/ip_output.c,v retrieving revision 1.99.2.13 diff -u -r1.99.2.13 ip_output.c --- netinet/ip_output.c 2001/03/11 22:18:00 1.99.2.13 +++ netinet/ip_output.c 2001/05/18 21:38:47 @@ -466,6 +466,21 @@ if (m) m_freem(m); error = EACCES ; +#if 1 + /* + * [for slice] ipfw_hook may claim the outgoing + * packet without generating an error. if + * ip_fw_chk_ptr returned our magic number, then + * return success. + * + * without this modification, slice packet + * redirection will generate "nfs send error 13" + * messages on the console. + */ + if (off & 0x30000) + error = 0; +#endif + goto done ; } ip = mtod(m, struct ip *); @@ -835,8 +850,50 @@ m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA; } + if (len > PAGE_SIZE) { + /* + * Fragement large datagrams such that each segment + * contains a multiple of PAGE_SIZE amount of data, + * plus headers. This enables a receiver to perform + * page-flipping zero-copy optimizations. + */ + + int newlen; + struct mbuf *mtmp; + + for (mtmp = m, off = 0; + mtmp && ((off + mtmp->m_len) <= ifp->if_mtu); + mtmp = mtmp->m_next) { + off += mtmp->m_len; + } + /* + * firstlen (off - hlen) must be aligned on an + * 8-byte boundary + */ + if (off < hlen) + goto smart_frag_failure; + off = ((off - hlen) & ~7) + hlen; + newlen = (~PAGE_MASK) & ifp->if_mtu; + if ((newlen + sizeof (struct ip)) > ifp->if_mtu) { + /* we failed, go back the default */ +smart_frag_failure: + newlen = len; + off = hlen + len; + } + +/* printf("ipfrag: len = %d, hlen = %d, mhlen = %d, newlen = %d, off = %d\n", + len, hlen, sizeof (struct ip), newlen, off);*/ + + len = newlen; + + } else { + off = hlen + len; + } + + + { - int mhlen, firstlen = len; + int mhlen, firstlen = off - hlen; struct mbuf **mnext = &m->m_nextpkt; int nfrags = 1; @@ -846,7 +903,7 @@ */ m0 = m; mhlen = sizeof (struct ip); - for (off = hlen + len; off < (u_short)ip->ip_len; off += len) { + for (; off < (u_short)ip->ip_len; off += len) { MGETHDR(m, M_DONTWAIT, MT_HEADER); if (m == 0) { error = ENOBUFS; Index: nfs/nfs_bio.c =================================================================== RCS file: /home/ncvs/src/sys/nfs/nfs_bio.c,v retrieving revision 1.83 diff -u -r1.83 nfs_bio.c --- nfs/nfs_bio.c 2000/01/05 05:11:36 1.83 +++ nfs/nfs_bio.c 2001/07/03 14:01:21 @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -320,6 +321,8 @@ return rtvals[0]; } +int nfs_read_cache_disable = 0; + /* * Vnode op for read using bio */ @@ -535,6 +538,12 @@ } } + if (nfs_read_cache_disable) { + /* no client caching - force re-read from remote */ + bp->b_flags |= B_NOCACHE; + bp->b_flags |= B_INVAL; + } + /* * on is the offset into the current bp. Figure out how many * bytes we can copy out of the bp. Note that bcount is @@ -682,7 +691,25 @@ }; if (n > 0) { - error = uiomove(bp->b_data + on, (int)n, uio); +#ifdef ENABLE_VFS_IOOPT + if (vfs_ioopt && vp->v_object && + (bp->b_flags & B_VMIO) && + ((on & PAGE_MASK) == 0) && + ((n & PAGE_MASK) == 0)) + /* + * If VFS IO optimisation is turned on, + * and it's an exact page multiple + * And a normal VM based op, + * then use uiomiveco() + */ + error = + uiomoveco((char *)bp->b_data + on, + (int)n, uio, vp->v_object, 0); + else + +#endif + error = uiomove(bp->b_data + on, (int)n, uio); + } switch (vp->v_type) { case VREG: @@ -1340,6 +1367,12 @@ * Do an I/O operation to/from a cache block. This may be called * synchronously or from an nfsiod. */ + +static int nfs_cluster_commit __P((struct vnode *vp, struct buf *bp, + struct proc *p)); +int nfs_zreadrpc __P((struct buf *, struct vnode *, struct uio *, + struct ucred *)); + int nfs_doio(bp, cr, p) struct buf *bp; @@ -1407,7 +1440,7 @@ case VREG: uiop->uio_offset = ((off_t)bp->b_blkno) * DEV_BSIZE; nfsstats.read_bios++; - error = nfs_readrpc(vp, uiop, cr); + error = nfs_zreadrpc(bp, vp, uiop, cr); if (!error) { if (uiop->uio_resid) { /* @@ -1473,24 +1506,10 @@ */ if (bp->b_flags & B_NEEDCOMMIT) { int retv; - off_t off; - off = ((u_quad_t)bp->b_blkno) * DEV_BSIZE + bp->b_dirtyoff; - bp->b_flags |= B_WRITEINPROG; - retv = nfs_commit( - bp->b_vp, off, bp->b_dirtyend-bp->b_dirtyoff, - bp->b_wcred, p); - bp->b_flags &= ~B_WRITEINPROG; - if (retv == 0) { - bp->b_dirtyoff = bp->b_dirtyend = 0; - bp->b_flags &= ~(B_NEEDCOMMIT | B_CLUSTEROK); - bp->b_resid = 0; - biodone(bp); + retv = nfs_cluster_commit(vp, bp, p); + if (retv == 0) return (0); - } - if (retv == NFSERR_STALEWRITEVERF) { - nfs_clearcommit(bp->b_vp->v_mount); - } } /* @@ -1591,3 +1610,203 @@ biodone(bp); return (error); } + + +#ifndef NFS_COMMITBVECSIZ +#define NFS_COMMITBVECSIZ 20 +#endif +int nfs_cluster_commitbufs = 2048; +/* + * nfs_cluster_commit(vp, bp, p) + * + * This function is called on a NFS buffer that needs a commit RPC + * Even though the buffer may already be clustered, clustering is limited + * to 64k chunks, try to grab an even bigger range by scanning forward and + * backward in the file. + */ +static int +nfs_cluster_commit(vp, bp, p) + struct vnode *vp; + struct buf *bp; + struct proc *p; +{ + struct buf **cbufs, *cbufs_on_stack[NFS_COMMITBVECSIZ], *holdbp, *testbp; + int commit_idx, retv, flags, pass, i, dirty_off, dirty_end, s, gap; + int lblocksize; + int maxcbufs; + int pre_count=0, post_count=0; + u_quad_t sblkno, eblkno, lblkno, olblkno; + /* start, end, logical block number */ + + KASSERT(bp->b_flags & B_NEEDCOMMIT, ("NFS commit without B_NEEDCOMMIT")); + + cbufs = NULL; + + if (nfs_cluster_commitbufs > NFS_COMMITBVECSIZ) { + cbufs = (struct buf **) + malloc(nfs_cluster_commitbufs * sizeof(struct buf *), + M_TEMP, M_NOWAIT); + } + + if (cbufs == NULL) { + cbufs = cbufs_on_stack; + maxcbufs = NFS_COMMITBVECSIZ; + } else { + maxcbufs = nfs_cluster_commitbufs; + } + flags = B_DELWRI | B_NEEDCOMMIT; + gap = 0; + commit_idx = 0; + pass = 0; + sblkno = eblkno = (u_quad_t)bp->b_lblkno; + dirty_end = bp->b_dirtyend; + dirty_off = bp->b_dirtyoff; + lblocksize = vp->v_mount->mnt_stat.f_iosize; + + bp->b_flags |= B_WRITEINPROG; + + s = splbio(); + /* + * first pass scan forward, second backwards from bp + */ +pass: + /* reinit the start location for our sweep */ + lblkno = (u_quad_t)bp->b_lblkno; + holdbp = bp; + + /* + * don't scan too much and don't overflow buf pointer array + */ + while (gap < 4 && commit_idx < maxcbufs) { + + /* scan forward or backward */ + if (pass == 1) { /* backwards */ + /* + * backwards is easy just subtracting one from the lbn + * should get us the previous buffer if it exists + */ + if (lblkno == 0) + break; + else + lblkno--; + } else { + /* + * when going forward make sure to remeber to account + * for buffers that span more than one block + * if holdbp == NULL then gbincore() failed, try + * going forward only a single block + */ + if (lblkno == (u_quad_t)-1) + break; + else if (holdbp == NULL) { + lblkno++; + } + else { + olblkno = lblkno; + lblkno += holdbp->b_bufsize / lblocksize; + if (olblkno == lblkno) + lblkno++; + } + } + + testbp = holdbp; + /* + * if the buffer isn't in memory bump the gap count + * and try the next one + */ + holdbp = gbincore(vp, lblkno); + if (holdbp == NULL) { + gap++; + continue; + } + if (testbp == holdbp) { + gap++; + continue; + } + + lblkno = (u_quad_t)holdbp->b_lblkno; + + /* + * make sure flags and cred are the same as the first buffer + * also make sure we can lock the buffer + * if not we have to escape the loop + */ + if (((holdbp->b_flags & flags) != flags) || + holdbp->b_wcred != bp->b_wcred || + BUF_LOCK(holdbp, LK_EXCLUSIVE | LK_NOWAIT)) + break; + + gap = 0; /* scan more */ + + bremfree(holdbp); /* remove from bufqueues */ + + holdbp->b_flags |= B_WRITEINPROG; + vfs_busy_pages(holdbp, 1); + cbufs[ commit_idx++ ] = holdbp; + if (pass == 1) { + /* going backwards so set start block and offset */ + sblkno = lblkno; + dirty_off = holdbp->b_dirtyoff; + ++pre_count; + } else { + /* going forward so set end block and end pointer */ + eblkno = lblkno; + dirty_end = holdbp->b_dirtyend; + ++post_count; + } + } + gap = 0; + if (++pass == 1) + goto pass; + + splx(s); + + retv = nfs_commit(bp->b_vp, + sblkno * lblocksize + dirty_off, + (eblkno - sblkno) * lblocksize + dirty_end, + bp->b_wcred, p); + + for (i = 0; i < commit_idx; i++) { + holdbp = cbufs[i]; + holdbp->b_flags &= ~(B_WRITEINPROG|B_NEEDCOMMIT); + if (retv) { + /* + * Error, leave B_DELWRI intact + */ + vfs_unbusy_pages(holdbp); + brelse(holdbp); + } else { + /* + * Success, remove B_DELWRI ( bundirty() ). + * + * b_dirtyoff/b_dirtyend seem to be NFS + * specific. We should probably move that + * into bundirty(). XXX + */ + s = splbio(); + vp->v_numoutput++; + holdbp->b_flags |= B_ASYNC; + bundirty(holdbp); + holdbp->b_flags &= ~(B_READ|B_DONE|B_ERROR); + holdbp->b_dirtyoff = holdbp->b_dirtyend = 0; + holdbp->b_resid = 0; + splx(s); + biodone(holdbp); + } + } + bp->b_flags &= ~B_WRITEINPROG; + if (retv == 0) { + bp->b_dirtyoff = bp->b_dirtyend = 0; + bp->b_flags &= ~B_NEEDCOMMIT; + bp->b_resid = 0; + biodone(bp); + } else if (retv == NFSERR_STALEWRITEVERF) { + nfs_clearcommit(bp->b_vp->v_mount); + } + + if (cbufs != cbufs_on_stack) + free(cbufs, M_TEMP); + + return (retv); + + } Index: nfs/nfs_serv.c =================================================================== RCS file: /home/ncvs/src/sys/nfs/nfs_serv.c,v retrieving revision 1.93 diff -u -r1.93 nfs_serv.c --- nfs/nfs_serv.c 1999/12/18 19:20:05 1.93 +++ nfs/nfs_serv.c 2001/05/18 14:23:01 @@ -99,6 +99,13 @@ #include #include +#include +#include +#include +extern struct sf_buf *sf_buf_alloc(void); +extern void sf_buf_ref(caddr_t addr, u_int size); +extern void sf_buf_free(caddr_t addr, u_int size); + #ifdef NFSRV_DEBUG #define nfsdbprintf(info) printf info #else @@ -755,6 +762,13 @@ /* * nfs read service */ + +static int nfs_serv_zcread = 1; +static int nfs_serv_zcreadyes = 0, nfs_serv_zcreadno = 0; +SYSCTL_INT(_vfs_nfs, OID_AUTO, serv_zcread, CTLFLAG_RW, &nfs_serv_zcread, 1, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, serv_zcread_count, CTLFLAG_RW, &nfs_serv_zcreadyes, 1, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, serv_cpread_count, CTLFLAG_RW, &nfs_serv_zcreadno, 1, ""); + int nfsrv_read(nfsd, slp, procp, mrq) struct nfsrv_descript *nfsd; @@ -919,54 +933,237 @@ } len = left = nfsm_rndup(cnt); if (cnt > 0) { - /* - * Generate the mbuf list with the uio_iov ref. to it. - */ - i = 0; - m = m2 = mb; - while (left > 0) { - siz = min(M_TRAILINGSPACE(m), left); - if (siz > 0) { - left -= siz; - i++; + +/* + * BEGIN SUSPECT REGION + * if offset is aligned and len is a multiple of page size, we should + * be able to use bread() to zero-copy read an appropriate struct buf. + * then wrap ext mbufs around it (final m_freem needs to unlock buf) + * and pass it along. hmm. + */ + if (nfs_serv_zcread) { + + /* preconditions: + * len = number of bytes to read. + * vp = valid vnode. + * mb is the header mbuf. + * m2 is a free temp mbuf. + * + * postconditions: + * mb->m_next points at a chain of mbufs containing the + * read data + * left <- 0 + * uiop->uio_offset <- 0 + * off <- nh->nextr <- offset where read ends + * error <- 0 + */ + + struct vm_object *obj; + struct sf_buf *sf; + struct vm_page *pg; + vm_pindex_t pindex; + vm_offset_t pgoff = 0; /* for now */ + int xfsize = PAGE_SIZE; /* for now */ + + m2 = mb; + + left = len; + if ((len & PAGE_SIZE) || (off & PAGE_SIZE)) { + /* + * don't try to read non-aligned offsets and/or + * non-page-sized lengths. + */ + goto normalpath; + } + + vref(vp); + obj = vp->v_object; + if (vp->v_type != VREG || obj == NULL) { + vrele(vp); + goto normalpath; } - if (left > 0) { - MGET(m, M_WAIT, MT_DATA); - MCLGET(m, M_WAIT); - m->m_len = 0; + + for (; left > 0 ; off += xfsize, left -= xfsize) { + pindex = OFF_TO_IDX(off); + + retry_lookup: + /* + * Attempt to look up the page. + * + * Allocate if not found + * + * Wait and loop if busy. + */ + pg = vm_page_lookup(obj, pindex); + + if (pg == NULL) { + pg = vm_page_alloc(obj, pindex, VM_ALLOC_NORMAL); + if (pg == NULL) { + VM_WAIT; + goto retry_lookup; + } + vm_page_wakeup(pg); + } else if (vm_page_sleep_busy(pg, TRUE, "sfpbsy")) { + goto retry_lookup; + } + + /* + * Wire the page so it does not get ripped out from under + * us. + */ + + vm_page_wire(pg); + + /* + * If page is not valid for what we need, initiate I/O + */ + + if (!pg->valid || !vm_page_is_valid(pg, pgoff, xfsize)) { + struct uio auio; + struct iovec aiov; + int bsize; + + /* + * Ensure that our page is still around when the I/O + * completes. + */ + vm_page_io_start(pg); + + /* + * Get the page from backing store. + */ + bsize = vp->v_mount->mnt_stat.f_iosize; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + aiov.iov_base = 0; + aiov.iov_len = MAXBSIZE; + auio.uio_resid = MAXBSIZE; + auio.uio_offset = trunc_page(off); + auio.uio_segflg = UIO_NOCOPY; + auio.uio_rw = UIO_READ; + auio.uio_procp = procp; + vn_lock(vp, LK_SHARED | LK_NOPAUSE | LK_RETRY, procp); + error = VOP_READ(vp, &auio, IO_VMIO | ((MAXBSIZE / bsize) << 16), + cred); + VOP_UNLOCK(vp, 0, procp); + vm_page_flag_clear(pg, PG_ZERO); + vm_page_io_finish(pg); + if (error) { + vm_page_unwire(pg, 0); + /* + * See if anyone else might know about this page. + * If not and it is not valid, then free it. + */ + if (pg->wire_count == 0 && pg->valid == 0 && + pg->busy == 0 && !(pg->flags & PG_BUSY) && + pg->hold_count == 0) { + if ((pg->flags & PG_BUSY) == 0) { + vm_page_busy(pg); + } + vm_page_free(pg); + } + goto errorpath; + } + } + + /* + * Allocate a kernel virtual page and insert the physical page + * into it. + */ + + sf = sf_buf_alloc(); + sf->m = pg; + pmap_qenter(sf->kva, &pg, 1); + /* + * Get an mbuf header and set it up as having external storage. + */ + MGETHDR(m, M_WAIT, MT_DATA); + if (m == NULL) { + error = ENOBUFS; + goto errorpath; + } + m->m_ext.ext_free = sf_buf_free; + m->m_ext.ext_ref = sf_buf_ref; + m->m_ext.ext_buf = (void *)sf->kva; + m->m_ext.ext_size = PAGE_SIZE; + m->m_data = (char *) sf->kva + pgoff; + m->m_flags |= M_EXT; + m->m_pkthdr.len = m->m_len = xfsize; + m2->m_next = m; m2 = m; } - } - MALLOC(iv, struct iovec *, i * sizeof (struct iovec), - M_TEMP, M_WAITOK); - uiop->uio_iov = iv2 = iv; - m = mb; - left = len; - i = 0; - while (left > 0) { - if (m == NULL) - panic("nfsrv_read iov"); - siz = min(M_TRAILINGSPACE(m), left); - if (siz > 0) { - iv->iov_base = mtod(m, caddr_t) + m->m_len; - iv->iov_len = siz; - m->m_len += siz; - left -= siz; - iv++; - i++; + + uiop->uio_resid = 0; + nh->nh_nextr = off; + error = 0; + + nfs_serv_zcreadyes++; + if (0) { + errorpath: + if (mb->m_next != NULL) { + m_freem(mb->m_next); + mb->m_next = 0; + } + vrele(vp); + goto normalpath; + } + + } else { + normalpath: + nfs_serv_zcreadno++; + + /* + * Generate the mbuf list with the uio_iov ref. to it. + */ + i = 0; + m = m2 = mb; + while (left > 0) { + siz = min(M_TRAILINGSPACE(m), left); + if (siz > 0) { + left -= siz; + i++; + } + if (left > 0) { + MGET(m, M_WAIT, MT_DATA); + MCLGET(m, M_WAIT); + m->m_len = 0; + m2->m_next = m; + m2 = m; + } + } + MALLOC(iv, struct iovec *, i * sizeof (struct iovec), + M_TEMP, M_WAITOK); + uiop->uio_iov = iv2 = iv; + m = mb; + left = len; + i = 0; + while (left > 0) { + if (m == NULL) + panic("nfsrv_read iov"); + siz = min(M_TRAILINGSPACE(m), left); + if (siz > 0) { + iv->iov_base = mtod(m, caddr_t) + m->m_len; + iv->iov_len = siz; + m->m_len += siz; + left -= siz; + iv++; + i++; + } + m = m->m_next; } - m = m->m_next; + uiop->uio_iovcnt = i; + uiop->uio_offset = off; + uiop->uio_resid = len; + uiop->uio_rw = UIO_READ; + uiop->uio_segflg = UIO_SYSSPACE; + error = VOP_READ(vp, uiop, IO_NODELOCKED | ioflag, cred); + off = uiop->uio_offset; + nh->nh_nextr = off; + FREE((caddr_t)iv2, M_TEMP); + } - uiop->uio_iovcnt = i; - uiop->uio_offset = off; - uiop->uio_resid = len; - uiop->uio_rw = UIO_READ; - uiop->uio_segflg = UIO_SYSSPACE; - error = VOP_READ(vp, uiop, IO_NODELOCKED | ioflag, cred); - off = uiop->uio_offset; - nh->nh_nextr = off; - FREE((caddr_t)iv2, M_TEMP); + if (error || (getret = VOP_GETATTR(vp, vap, cred, procp))) { if (!error) error = getret; Index: nfs/nfs_socket.c =================================================================== RCS file: /home/ncvs/src/sys/nfs/nfs_socket.c,v retrieving revision 1.60.2.1 diff -u -r1.60.2.1 nfs_socket.c --- nfs/nfs_socket.c 2000/03/27 21:39:53 1.60.2.1 +++ nfs/nfs_socket.c 2001/05/01 20:46:27 @@ -1450,7 +1450,7 @@ ((nmp->nm_flag & NFSMNT_DUMBTIMR) || (rep->r_flags & R_SENT) || nmp->nm_sent < nmp->nm_cwnd) && - (m = m_copym(rep->r_mreq, 0, M_COPYALL, M_DONTWAIT))){ + (m = m_dup(rep->r_mreq, M_DONTWAIT))){ if ((nmp->nm_flag & NFSMNT_NOCONN) == 0) error = (*so->so_proto->pr_usrreqs->pru_send) (so, 0, m, (struct sockaddr *)0, Index: nfs/nfs_subs.c =================================================================== RCS file: /home/ncvs/src/sys/nfs/nfs_subs.c,v retrieving revision 1.90.2.1 diff -u -r1.90.2.1 nfs_subs.c --- nfs/nfs_subs.c 2000/10/28 16:27:27 1.90.2.1 +++ nfs/nfs_subs.c 2001/05/18 14:58:25 @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -75,6 +76,8 @@ #include +#include + /* * Data items converted to xdr at startup, since they are constant * This is kinda hokey, but may save a little time doing byte swaps @@ -776,6 +779,197 @@ } /* + * copies mbuf chain to the uio scatter/gather list, moving pages if possible + */ + + +static int nfs_client_zcread = 1; +SYSCTL_DECL(_vfs_nfs); +SYSCTL_INT(_vfs_nfs, OID_AUTO, client_zcread, CTLFLAG_RW, &nfs_client_zcread, 1, ""); + +int nfsm_zmbuftouio __P((struct buf *bp, struct mbuf **, struct uio *, int, caddr_t *)); + +#define STAT_INC(W,X) (W.X)++ +struct znfs_stats { + int mapin; + int copyin; + int cantmap_paging, cantmap_busy, cantmap_locked, + cantmap_bogus, cantmap_chain, cantmap_toosmall, + cantmap_mbufcp, cantmap_len; +} znfs_stats; + +#include +#include +extern vm_page_t bogus_page; + +int +nfsm_zmbuftouio(bp, mrep, uiop, siz, dpos) + struct buf *bp; /* struct buf we are copying to */ + struct mbuf **mrep; /* start of mbuf chain */ + register struct uio *uiop; /* uio struct */ + int siz; /* how much data to copy */ + caddr_t *dpos; /* pointer into mbuf chain to copy */ +{ + register char *mbufcp; + register int len, total; + register struct mbuf *mp; + long rem; + int error = 0, npages = 0; + int ok, s, i, busy = bp->b_pages[0]->busy; + vm_object_t object = bp->b_vp->v_object; + vm_page_t pages[16]; + + mp = *mrep; + mbufcp = *dpos; + len = mtod(mp, caddr_t)+mp->m_len-mbufcp; + rem = nfsm_rndup(siz)-siz; + + s = splvm(); + if (nfs_client_zcread == 0) + goto cantmap; + + if (siz < PAGE_SIZE) { + STAT_INC(znfs_stats, cantmap_toosmall); + goto cantmap; + } + + /* + * oddly enough, len will be 0 when we start the ball rolling + * because the rpc code will have plucked the nfs goop off the + * front of the mbuf chain. Make sure this is so + */ + + if (len != 0) { + STAT_INC(znfs_stats, cantmap_len); + } + /* + * walk the mbuf chain, gathering up M_EXT mbufs & deciding if + * we can do this easily. In the future, we should allow for + * the last mbuf in the chain to be < PAGE_SIZE + */ + total = 0; + ok = 1; + npages = 0; + + while (mp && (total < siz) && ok) { + len = 0; + while (len == 0) { + mp = mp->m_next; + if (mp == NULL) + return (EBADRPC); + mbufcp = mtod(mp, caddr_t); + len = mp->m_len; + } + total += len; + ok = mbufcp && ((len == PAGE_SIZE ) && + (((vm_offset_t)mbufcp & PAGE_MASK) == 0)); + + if (ok) + pages[npages++] = + PHYS_TO_VM_PAGE(pmap_extract(kernel_pmap, + (vm_offset_t)mbufcp)); + } + if (!ok) { + STAT_INC(znfs_stats, cantmap_chain); + if ((vm_offset_t)mbufcp & PAGE_MASK) + STAT_INC(znfs_stats, cantmap_mbufcp); + if (len != PAGE_SIZE) + STAT_INC(znfs_stats, cantmap_len); + goto cantmap; + } + /* now we know that we have an mbuf chain with page-sized, + page-aligned, ext mbufs. Go for it !! */ + + if (bp->b_flags & B_PAGING) { + /* + * could map, but need to do LOTS more. + * (user pmap needs tweaking!) XXX + */ + STAT_INC(znfs_stats, cantmap_paging); + goto cantmap; + } + for (i=0 ; ib_pages[i]; + if (m == bogus_page) { + STAT_INC(znfs_stats, cantmap_bogus); + goto cantmap; + } +#if !defined(MAX_PERF) + if (m->busy != busy) + panic("busy count mismatch"); +#endif + if ((m->flags & PG_WANTED) || (m->wire_count > 1) || + (m->busy > 1)) { + STAT_INC(znfs_stats, cantmap_busy); + goto cantmap; + } + } + + STAT_INC(znfs_stats, mapin); + + /* + * disassociate existing pages from buf (still splvm) + */ + pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages); + for (i=0 ; ib_pages[i]; + + vm_page_busy(pages[i]); + vm_page_remove(pages[i]); /* remove page from the trapeze object */ + + pages[i]->pindex = m->pindex; /* remember pindex */ + + if (busy) + vm_page_io_finish(m); + vm_page_unwire(m, 0); + vm_page_busy(m); + /* + * XXX put this back + * when mmap (B_PAGING) is supported + */ +/* vm_page_protect(m, VM_PROT_NONE); */ + vm_page_free(m); + } + + /* + * map new pages into buf (still splvm); don't assumes pages were + * vm_page_remove()ed earlier. + */ + for (i=0 ; ipindex); + vm_page_flag_clear(m, PG_ZERO); + vm_page_wire(m); + if (busy) + vm_page_io_start(m); + + bp->b_pages[i] = m; + } + + pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages); + splx(s); + + uiop->uio_resid = 0; + + /* + * XXX we should really fiddle with the uio & advance the + * pointers into the mbuf chain. The only thing we need to do + * is to clear the uiop->uio_resid; this needs to be done to prevent + * doio() from seeing what we did as a short read with no error + * and zero-filling the file. + */ + + return error; + + + /* NOTREACHED */ +cantmap: + splx(s); + return nfsm_mbuftouio(mrep, uiop, siz, dpos); +} + +/* * copies mbuf chain to the uio scatter/gather list */ int @@ -850,6 +1044,48 @@ return (error); } +static int +znfsm_uiotombuf(struct uio *uiop, struct mbuf **mq, int siz, caddr_t *bpos) +{ + register struct mbuf *mp, *mp2; + register int left; + int uiosiz, rem; + char *cp; + + rem = nfsm_rndup(siz)-siz; + uiosiz = uiop->uio_iov->iov_len; + if (uiosiz > siz) + uiosiz = siz; + mp2 = *mq; + mp = emb_get(uiop->uio_iov->iov_base, uiosiz, NULL, 0, M_WAIT); + if (mp == NULL) + return 1; + mp2->m_next = mp; + uiop->uio_iov->iov_base += uiosiz; + uiop->uio_iov->iov_len -= uiosiz; + uiop->uio_offset += uiosiz; + uiop->uio_resid -= uiosiz; + + while(mp->m_next != NULL) { + mp = mp->m_next; + } + if (rem > 0) { + mp2 = mp; + MGET(mp, M_WAIT, MT_DATA); + mp->m_len = 0; + mp2->m_next = mp; + cp = mtod(mp, caddr_t)+mp->m_len; + for (left = 0; left < rem; left++) + *cp++ = '\0'; + mp->m_len += rem; + *bpos = cp; + } else + *bpos = mtod(mp, caddr_t)+mp->m_len; + *mq = mp; + return (0); +} + +int nfswritehack = 1; /* * copies a uio scatter/gather list to an mbuf chain. * NOTE: can ony handle iovcnt == 1 @@ -876,6 +1112,20 @@ clflg = 1; else clflg = 0; + + if (nfswritehack && clflg && uiop->uio_segflg == UIO_SYSSPACE) { +/*printf("BEFORE: size = %d, bpos = %p, rem = %d, offset = %d, resid = %d\n", siz, bpos, + nfsm_rndup(siz)-siz, uiop->uio_offset, uiop->uio_resid);*/ + if(nfswritehack==2) goto normal; + if (0 == (znfsm_uiotombuf(uiop, mq, siz, bpos))) + { +/*if(nfswritehack) printf("AFTER: bpos = %p, iov_base = %p, iov_len = %d, offset = %d, resid = %d\n", + bpos, uiop->uio_iov->iov_base, uiop->uio_iov->iov_len, uiop->uio_offset,uiop->uio_resid );*/ + + return (0); + } + } +normal: rem = nfsm_rndup(siz)-siz; mp = mp2 = *mq; while (siz > 0) { @@ -931,6 +1181,8 @@ } else *bpos = mtod(mp, caddr_t)+mp->m_len; *mq = mp; +/*if(nfswritehack) printf("AFTER: bpos = %p, iov_base = %p, iov_len = %d, offset = %d, resid = %d\n", + bpos, uiop->uio_iov->iov_base, uiop->uio_iov->iov_len, uiop->uio_offset,uiop->uio_resid );*/ return (0); } Index: nfs/nfs_vnops.c =================================================================== RCS file: /home/ncvs/src/sys/nfs/nfs_vnops.c,v retrieving revision 1.150.2.2 diff -u -r1.150.2.2 nfs_vnops.c --- nfs/nfs_vnops.c 2001/03/02 16:45:12 1.150.2.2 +++ nfs/nfs_vnops.c 2001/05/18 14:50:09 @@ -1061,6 +1061,79 @@ * nfs read rpc call * Ditto above */ +int nfs_zreadrpc __P((struct buf *, struct vnode *, struct uio *, struct ucred *)); +int nfsm_zmbuftouio __P((struct buf *bp, struct mbuf **, struct uio *, int, caddr_t *)); + +int +nfs_zreadrpc(bp, vp, uiop, cred) + struct buf *bp; + register struct vnode *vp; + struct uio *uiop; + struct ucred *cred; +{ + register u_int32_t *tl; + register caddr_t cp; + register int32_t t1, t2; + caddr_t bpos, dpos, cp2; + struct mbuf *mreq, *mrep, *md, *mb, *mb2; + struct nfsmount *nmp; + int error = 0, len, retlen, tsiz, eof, attrflag; + int v3 = NFS_ISV3(vp); + +#ifndef nolint + eof = 0; +#endif + nmp = VFSTONFS(vp->v_mount); + tsiz = uiop->uio_resid; + if (uiop->uio_offset + tsiz > nmp->nm_maxfilesize) + return (EFBIG); + while (tsiz > 0) { + nfsstats.rpccnt[NFSPROC_READ]++; + len = (tsiz > nmp->nm_rsize) ? nmp->nm_rsize : tsiz; + nfsm_reqhead(vp, NFSPROC_READ, NFSX_FH(v3) + NFSX_UNSIGNED * 3); + nfsm_fhtom(vp, v3); + nfsm_build(tl, u_int32_t *, NFSX_UNSIGNED * 3); + if (v3) { + txdr_hyper(uiop->uio_offset, tl); + *(tl + 2) = txdr_unsigned(len); + } else { + *tl++ = txdr_unsigned(uiop->uio_offset); + *tl++ = txdr_unsigned(len); + *tl = 0; + } + nfsm_request(vp, NFSPROC_READ, uiop->uio_procp, cred); + if (v3) { + nfsm_postop_attr(vp, attrflag); + if (error) { + m_freem(mrep); + goto nfsmout; + } + nfsm_dissect(tl, u_int32_t *, 2 * NFSX_UNSIGNED); + eof = fxdr_unsigned(int, *(tl + 1)); + } else + nfsm_loadattr(vp, (struct vattr *)0); + nfsm_strsiz(retlen, nmp->nm_rsize); + if ((retlen) > 0 && (t1 = + nfsm_zmbuftouio(bp, &md, uiop, retlen, &dpos)) != 0) { + error = t1; + m_freem(mrep); + goto nfsmout; + } + m_freem(mrep); + tsiz -= retlen; + if (v3) { + if (eof || retlen == 0) + tsiz = 0; + } else if (retlen < len) + tsiz = 0; + } +nfsmout: + return (error); +} +/* + * nfs read rpc call + * Ditto above + */ int nfs_readrpc(vp, uiop, cred) register struct vnode *vp; Index: nfs/nfsproto.h =================================================================== RCS file: /home/ncvs/src/sys/nfs/nfsproto.h,v retrieving revision 1.7 diff -u -r1.7 nfsproto.h --- nfs/nfsproto.h 1999/08/28 00:50:03 1.7 +++ nfs/nfsproto.h 2001/05/01 20:38:46 @@ -56,7 +56,7 @@ #define NFS_VER2 2 #define NFS_VER3 3 #define NFS_V2MAXDATA 8192 -#define NFS_MAXDGRAMDATA 16384 +#define NFS_MAXDGRAMDATA 32768 #define NFS_MAXDATA 32768 #define NFS_MAXPATHLEN 1024 #define NFS_MAXNAMLEN 255 Index: pci/if_ti.c =================================================================== RCS file: /home/ncvs/src/sys/pci/if_ti.c,v retrieving revision 1.25.2.7 diff -u -r1.25.2.7 if_ti.c --- pci/if_ti.c 2000/08/24 00:07:58 1.25.2.7 +++ pci/if_ti.c 2001/05/17 15:26:04 @@ -88,6 +88,8 @@ #include #include #include +/*#include */ /* XXX KDM */ +#include /* XXX KDM */ #include #include @@ -115,32 +117,48 @@ #include #include +/* #define PRIVATE_JUMBOS */ + +#if !defined(PRIVATE_JUMBOS) +#include +#endif /* !PRIVATE_JUMBOS */ +#include /* for vfindev, vgone */ + #include #include +#include #include #include #include + +#define TI_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_IP_FRAGS) +#define TI_JUMBO_HDRSPLIT + /* - * Temporarily disable the checksum offload support for now. - * Tests with ftp.freesoftware.com show that after about 12 hours, - * the firmware will begin calculating completely bogus TX checksums - * and refuse to stop until the interface is reset. Unfortunately, - * there isn't enough time to fully debug this before the 4.1 - * release, so this will need to stay off for now. + * We can only turn on header splitting if we're using extended receive + * BDs. */ -#ifdef notdef -#define TI_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_IP_FRAGS) -#else -#define TI_CSUM_FEATURES 0 -#endif +#if defined(TI_JUMBO_HDRSPLIT) && defined(PRIVATE_JUMBOS) +#error "options TI_JUMBO_HDRSPLIT and PRIVATE_JUMBOS are mutually exclusive" +#endif /* TI_JUMBO_HDRSPLIT && TI_JUMBO_HDRSPLIT */ + #if !defined(lint) static const char rcsid[] = - "$FreeBSD: src/sys/pci/if_ti.c,v 1.25.2.7 2000/08/24 00:07:58 wpaul Exp $"; + "$FreeBSD: src/sys/pci/if_ti.c,v 1.25 2000/01/18 00:26:29 wpaul Exp $"; #endif +struct ti_softc *tis[8]; + +typedef enum { + TI_SWAP_HTON, + TI_SWAP_NTOH +} ti_swap_type; + + + /* * Various supported device vendors/types and their names. */ @@ -163,6 +181,30 @@ { 0, 0, NULL } }; + +#define TI_CDEV_MAJOR 149 + +static d_open_t ti_open; +static d_close_t ti_close; +static d_ioctl_t ti_ioctl2; + +static struct cdevsw ti_cdevsw = { + /* open */ ti_open, + /* close */ ti_close, + /* read */ NULL, + /* write */ NULL, + /* ioctl */ ti_ioctl2, + /* poll */ seltrue, + /* mmap */ nommap, + /* strategy */ nostrategy, + /* name */ "ti", + /* maj */ TI_CDEV_MAJOR, + /* dump */ nodump, + /* psize */ nopsize, + /* flags */ 0, + /* bmaj */ -1 +}; + static int ti_probe __P((device_t)); static int ti_attach __P((device_t)); static int ti_detach __P((device_t)); @@ -195,15 +237,23 @@ static void ti_mem __P((struct ti_softc *, u_int32_t, u_int32_t, caddr_t)); +static int ti_copy_mem __P((struct ti_softc *, u_int32_t, + u_int32_t, caddr_t, int, int)); +static int ti_copy_scratch __P((struct ti_softc *, u_int32_t, + u_int32_t, caddr_t, int, int, int)); +static int ti_bcopy_swap __P((const void *, void *, size_t, + ti_swap_type)); static void ti_loadfw __P((struct ti_softc *)); static void ti_cmd __P((struct ti_softc *, struct ti_cmd_desc *)); static void ti_cmd_ext __P((struct ti_softc *, struct ti_cmd_desc *, caddr_t, int)); static void ti_handle_events __P((struct ti_softc *)); +#ifdef PRIVATE_JUMBOS static int ti_alloc_jumbo_mem __P((struct ti_softc *)); static void *ti_jalloc __P((struct ti_softc *)); static void ti_jfree __P((caddr_t, u_int)); static void ti_jref __P((caddr_t, u_int)); +#endif /* PRIVATE_JUMBOS */ static int ti_newbuf_std __P((struct ti_softc *, int, struct mbuf *)); static int ti_newbuf_mini __P((struct ti_softc *, int, struct mbuf *)); static int ti_newbuf_jumbo __P((struct ti_softc *, int, struct mbuf *)); @@ -239,6 +289,21 @@ DRIVER_MODULE(if_ti, pci, ti_driver, ti_devclass, 0, 0); + +/* List of Tigon softcs */ +static STAILQ_HEAD(ti_softc_list, ti_softc) ti_sc_list; + +static struct ti_softc * +ti_lookup_softc(int unit) +{ + struct ti_softc *sc; + for (sc = STAILQ_FIRST(&ti_sc_list); sc != NULL; + sc = STAILQ_NEXT(sc, ti_links)) + if (sc->ti_unit == unit) + return(sc); + return(NULL); +} + /* * Send an instruction or address to the EEPROM, check for ACK. */ @@ -419,6 +484,341 @@ return; } +static int +ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata) + struct ti_softc *sc; + u_int32_t tigon_addr, len; + caddr_t buf; + int useraddr, readdata; +{ + int segptr, segsize, cnt; + caddr_t ptr; + int s; + u_int32_t origwin; + u_int8_t tmparray[TI_WINLEN], tmparray2[TI_WINLEN]; + int resid, segresid; + int first_pass; + + /* + * At the moment, we don't handle non-aligned cases, we just bail. + * If this proves to be a problem, it will be fixed. + */ + if ((readdata == 0) + && (tigon_addr & 0x3)) { + printf("ti%d: ti_copy_mem: tigon address %#x isn't " + "word-aligned\n", sc->ti_unit, tigon_addr); + printf("ti%d: ti_copy_mem: unaligned writes aren't yet " + "supported\n", sc->ti_unit); + return(EINVAL); + } + + segptr = tigon_addr & ~0x3; + segresid = tigon_addr - segptr; + + /* + * This is the non-aligned amount left over that we'll need to + * copy. + */ + resid = len & 0x3; + + /* Add in the left over amount at the front of the buffer */ + resid += segresid; + + cnt = len & ~0x3; + /* + * If resid + segresid is >= 4, add multiples of 4 to the count and + * decrease the residual by that much. + */ + cnt += resid & ~0x3; + resid -= resid & ~0x3; + + ptr = buf; + + first_pass = 1; + + /* + * Make sure we aren't interrupted while we're changing the window + * pointer. + */ + s = splimp(); + + /* + * Save the old window base value. + */ + origwin = CSR_READ_4(sc, TI_WINBASE); + + while(cnt) { + bus_size_t ti_offset; + + if (cnt < TI_WINLEN) + segsize = cnt; + else + segsize = TI_WINLEN - (segptr % TI_WINLEN); + CSR_WRITE_4(sc, TI_WINBASE, (segptr & ~(TI_WINLEN - 1))); + + ti_offset = TI_WINDOW + (segptr & (TI_WINLEN -1)); + + if (readdata) { + + bus_space_read_region_4(sc->ti_btag, + sc->ti_bhandle, ti_offset, + (u_int32_t *)tmparray, + segsize >> 2); + if (useraddr) { + /* + * Yeah, this is a little on the kludgy + * side, but at least this code is only + * used for debugging. + */ + ti_bcopy_swap(tmparray, tmparray2, segsize, + TI_SWAP_NTOH); + + if (first_pass) { + copyout(&tmparray2[segresid], ptr, + segsize - segresid); + first_pass = 0; + } else + copyout(tmparray2, ptr, segsize); + } else { + if (first_pass) { + + ti_bcopy_swap(tmparray, tmparray2, + segsize, TI_SWAP_NTOH); + bcopy(&tmparray2[segresid], ptr, + segsize - segresid); + first_pass = 0; + } else + ti_bcopy_swap(tmparray, ptr, segsize, + TI_SWAP_NTOH); + } + + } else { + if (useraddr) { + copyin(ptr, tmparray2, segsize); + ti_bcopy_swap(tmparray2, tmparray, segsize, + TI_SWAP_HTON); + } else + ti_bcopy_swap(ptr, tmparray, segsize, + TI_SWAP_HTON); + + bus_space_write_region_4(sc->ti_btag, + sc->ti_bhandle, ti_offset, + (u_int32_t *)tmparray, + segsize >> 2); + } + segptr += segsize; + ptr += segsize; + cnt -= segsize; + } + + /* + * Handle leftover, non-word-aligned bytes. + */ + if (resid != 0) { + u_int32_t tmpval, tmpval2; + bus_size_t ti_offset; + + /* + * Set the segment pointer. + */ + CSR_WRITE_4(sc, TI_WINBASE, (segptr & ~(TI_WINLEN - 1))); + + ti_offset = TI_WINDOW + (segptr & (TI_WINLEN - 1)); + + /* + * First, grab whatever is in our source/destination. + * We'll obviously need this for reads, but also for + * writes, since we'll be doing read/modify/write. + */ + bus_space_read_region_4(sc->ti_btag, sc->ti_bhandle, + ti_offset, &tmpval, 1); + + /* + * Next, translate this from little-endian to big-endian + * (at least on i386 boxes). + */ + tmpval2 = ntohl(tmpval); + + if (readdata) { + /* + * If we're reading, just copy the leftover number + * of bytes from the host byte order buffer to + * the user's buffer. + */ + if (useraddr) + copyout(&tmpval2, ptr, resid); + else + bcopy(&tmpval2, ptr, resid); + } else { + /* + * If we're writing, first copy the bytes to be + * written into the network byte order buffer, + * leaving the rest of the buffer with whatever was + * originally in there. Then, swap the bytes + * around into host order and write them out. + * + * XXX KDM the read side of this has been verified + * to work, but the write side of it has not been + * verified. So user beware. + */ + if (useraddr) + copyin(ptr, &tmpval2, resid); + else + bcopy(ptr, &tmpval2, resid); + + tmpval = htonl(tmpval2); + + bus_space_write_region_4(sc->ti_btag, sc->ti_bhandle, + ti_offset, &tmpval, 1); + } + } + + CSR_WRITE_4(sc, TI_WINBASE, origwin); + + splx(s); + + return(0); +} + +static int +ti_copy_scratch(sc, tigon_addr, len, buf, useraddr, readdata, cpu) + struct ti_softc *sc; + u_int32_t tigon_addr, len; + caddr_t buf; + int useraddr, readdata; + int cpu; +{ + u_int32_t segptr; + int s, cnt; + u_int32_t tmpval, tmpval2; + caddr_t ptr; + + /* + * At the moment, we don't handle non-aligned cases, we just bail. + * If this proves to be a problem, it will be fixed. + */ + if (tigon_addr & 0x3) { + printf("ti%d: ti_copy_scratch: tigon address %#x isn't " + "word-aligned\n", sc->ti_unit, tigon_addr); + return(EINVAL); + } + + if (len & 0x3) { + printf("ti%d: ti_copy_scratch: transfer length %d isn't " + "word-aligned\n", sc->ti_unit, len); + return(EINVAL); + } + + segptr = tigon_addr; + cnt = len; + ptr = buf; + + s = splimp(); + + while (cnt) { + CSR_WRITE_4(sc, CPU_REG(TI_SRAM_ADDR, cpu), segptr); + + if (readdata) { + tmpval2 = CSR_READ_4(sc, CPU_REG(TI_SRAM_DATA, cpu)); + + tmpval = ntohl(tmpval2); + + /* + * Note: I've used this debugging interface + * extensively with Alteon's 12.3.15 firmware, + * compiled with GCC 2.7.2.1 and binutils 2.9.1. + * + * When you compile the firmware without + * optimization, which is necessary sometimes in + * order to properly step through it, you sometimes + * read out a bogus value of 0xc0017c instead of + * whatever was supposed to be in that scratchpad + * location. That value is on the stack somewhere, + * but I've never been able to figure out what was + * causing the problem. + * + * The address seems to pop up in random places, + * often not in the same place on two subsequent + * reads. + * + * In any case, the underlying data doesn't seem + * to be affected, just the value read out. + * + * KDM, 3/7/2000 + */ + + if (tmpval2 == 0xc0017c) + printf("ti%d: found 0xc0017c at %#x " + "(tmpval2)\n", sc->ti_unit, segptr); + + if (tmpval == 0xc0017c) + printf("ti%d: found 0xc0017c at %#x " + "(tmpval)\n", sc->ti_unit, segptr); + + if (useraddr) + copyout(&tmpval, ptr, 4); + else + bcopy(&tmpval, ptr, 4); + } else { + if (useraddr) + copyin(ptr, &tmpval2, 4); + else + bcopy(ptr, &tmpval2, 4); + + tmpval = htonl(tmpval2); + + CSR_WRITE_4(sc, CPU_REG(TI_SRAM_DATA, cpu), tmpval); + } + + cnt -= 4; + segptr += 4; + ptr += 4; + } + + splx(s); + + return(0); +} + +static int +ti_bcopy_swap(src, dst, len, swap_type) + const void *src; + void *dst; + size_t len; + ti_swap_type swap_type; +{ + const u_int8_t *tmpsrc; + u_int8_t *tmpdst; + size_t tmplen; + + if (len & 0x3) { + printf("ti_bcopy_swap: length %d isn't 32-bit aligned\n", + len); + return(-1); + } + + tmpsrc = src; + tmpdst = dst; + tmplen = len; + + while (tmplen) { + if (swap_type == TI_SWAP_NTOH) + *(u_int32_t *)tmpdst = + ntohl(*(const u_int32_t *)tmpsrc); + else + *(u_int32_t *)tmpdst = + htonl(*(const u_int32_t *)tmpsrc); + + tmpsrc += 4; + tmpdst += 4; + tmplen -= 4; + } + + return(0); +} + + + /* * Load firmware image into the NIC. Check that the firmware revision * is acceptable and see if we want the firmware for the Tigon 1 or @@ -584,6 +984,8 @@ return; } +#ifdef PRIVATE_JUMBOS + /* * Memory management for the jumbo receive ring is a pain in the * butt. We need to allocate at least 9018 bytes of space per frame, @@ -768,6 +1170,7 @@ return; } +#endif /* PRIVATE_JUMBOS */ /* * Intialize a standard receive ring descriptor. @@ -856,6 +1259,8 @@ return(0); } +#ifdef PRIVATE_JUMBOS + /* * Initialize a jumbo receive ring descriptor. This allocates * a jumbo buffer from the pool managed internally by the driver. @@ -916,6 +1321,161 @@ return(0); } +#else + +#if (PAGE_SIZE == 4096) +#define NPAYLOAD 2 +#else +#define NPAYLOAD 1 +#endif + +#define TCP_HDR_LEN (52 + sizeof(struct ether_header)) +#define UDP_HDR_LEN (28 + sizeof(struct ether_header)) +#define NFS_HDR_LEN (UDP_HDR_LEN) +int HDR_LEN = TCP_HDR_LEN; + + + /* + * Initialize a jumbo receive ring descriptor. This allocates + * a jumbo buffer from the pool managed internally by the driver. + */ +static int +ti_newbuf_jumbo(sc, idx, m_old) + struct ti_softc *sc; + int idx; + struct mbuf *m_old; +{ + struct mbuf *cur, *m_new = NULL; + struct mbuf *m[3] = {NULL, NULL, NULL}; + struct ti_rx_desc_ext *r; + vm_page_t frame; + /* 1 extra buf to make nobufs easy*/ + caddr_t buf[3] = {NULL, NULL, NULL}; + int i, s; + + if (m_old != NULL) { + m_new = m_old; + cur = m_old->m_next; + for(i = 0; i <= NPAYLOAD; i++){ + m[i] = cur; + cur = cur->m_next; + } + + } else { + +/* + * splhigh to avoid splimp/splx in mbuf code & splvm/splx in jumbo_pg_alloc + */ + s = splhigh(); + /* Allocate the mbufs. */ + MGETHDR(m_new, M_DONTWAIT, MT_DATA); + if (m_new == NULL) { + printf("ti%d: mbuf allocation failed " + "-- packet dropped!\n", sc->ti_unit); + goto nobufs; + } + MGET(m[NPAYLOAD], M_DONTWAIT, MT_DATA); + if (m[NPAYLOAD] == NULL) { + printf("ti%d: cluster mbuf allocation failed " + "-- packet dropped!\n", sc->ti_unit); + goto nobufs; + } + MCLGET(m[NPAYLOAD], M_DONTWAIT); + if ((m[NPAYLOAD]->m_flags & M_EXT) == 0) { + printf("ti%d: mbuf allocation failed " + "-- packet dropped!\n", sc->ti_unit); + goto nobufs; + } + m[NPAYLOAD]->m_len = MCLBYTES; + + for(i = 0; i < NPAYLOAD; i++){ + MGET(m[i], M_DONTWAIT, MT_DATA); + if (m[i] == NULL) { + printf("ti%d: mbuf allocation failed " + "-- packet dropped!\n", sc->ti_unit); + goto nobufs; + } + if(!(frame = jumbo_pg_alloc())){ + printf("ti%d: buffer allocation failed " + "-- packet dropped!", sc->ti_unit); + printf(" index %d page %d\n", idx, i); + goto nobufs; + } + buf[i] = jumbo_phys_to_kva(VM_PAGE_TO_PHYS(frame)); + } + splx(s); + for(i = 0; i < NPAYLOAD; i++){ + /* Attach the buffer to the mbuf. */ + m[i]->m_data = m[i]->m_ext.ext_buf = (void *)buf[i]; + m[i]->m_flags |= M_EXT; + m[i]->m_ext.ext_size = m[i]->m_len = PAGE_SIZE; + m[i]->m_ext.ext_free = jumbo_freem; + m[i]->m_ext.ext_ref = jumbo_ref; + m[i]->m_next = m[i+1]; + } + /* link the buffers to the header */ + m_new->m_next = m[0]; + m_new->m_data += ETHER_ALIGN; +#ifdef TI_JUMBO_HDRSPLIT + m_new->m_len = MHLEN - ETHER_ALIGN;; +#else /* TI_JUMBO_HDRSPLIT */ + m_new->m_len = HDR_LEN; +#endif /* TI_JUMBO_HDRSPLIT */ + m_new->m_pkthdr.len = NPAYLOAD * PAGE_SIZE + m_new->m_len; + + } + + /* Set up the descriptor. */ + r = &sc->ti_rdata->ti_rx_jumbo_ring[idx]; + sc->ti_cdata.ti_rx_jumbo_chain[idx] = m_new; + + TI_HOSTADDR(r->ti_addr0) = vtophys(mtod(m_new, caddr_t)); + r->ti_len0 = m_new->m_len; + + TI_HOSTADDR(r->ti_addr1) = vtophys(mtod(m[0], caddr_t)); + r->ti_len1 = PAGE_SIZE; + + TI_HOSTADDR(r->ti_addr2) = vtophys(mtod(m[1], caddr_t)); + r->ti_len2 = m[1]->m_ext.ext_size; /* could be PAGE_SIZE or MCLBYTES */ + + if (PAGE_SIZE == 4096) { + TI_HOSTADDR(r->ti_addr3) = vtophys(mtod(m[2], caddr_t)); + r->ti_len3 = MCLBYTES; + } else { + r->ti_len3 = 0; + } + r->ti_type = TI_BDTYPE_RECV_JUMBO_BD; + + r->ti_flags = TI_BDFLAG_JUMBO_RING|TI_RCB_FLAG_USE_EXT_RX_BD; + if (sc->arpcom.ac_if.if_hwassist) + r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM; + r->ti_idx = idx; + + return(0); + + nobufs: +/* + * Warning! : + * This can only be called before the mbufs are strung together. + * If the mbufs are strung together, m_freem() will free the chain, + * so that the later mbufs will be freed multiple times. + */ + splx(s); + if(m_new) + m_freem(m_new); + + for(i = 0; i < 3; i++){ + if(m[i]) + m_freem(m[i]); + if(buf[i]) + jumbo_pg_free((vm_offset_t)buf[i]); + } + return ENOBUFS; +} +#endif + + + /* * The standard receive ring has 512 entries in it. At 2K per mbuf cluster, * that's 1MB or memory, which is a lot. For now, we fill only the first @@ -962,7 +1522,22 @@ register int i; struct ti_cmd_desc cmd; + /* + * It seems that you need a sort of window between the number of + * jumbo buffers allocated and the number stuck into the receive + * ring. If the number of buffers allocated to the receive ring is + * too high, the driver will end up running out of new buffers to + * put in because the old buffers aren't getting freed fast enough. + * Yes, I know this explanation doesn't make a whole lot of sense, + * but I don't feel like taking the time to think it through + * further. + * KDM, 8/12/99 + */ +#ifdef PRIVATE_JUMBOS + for (i = 0; i < (TI_JSLOTS - 40); i++) { +#else for (i = 0; i < TI_JUMBO_RX_RING_CNT; i++) { +#endif if (ti_newbuf_jumbo(sc, i, NULL) == ENOBUFS) return(ENOBUFS); }; @@ -1213,6 +1788,7 @@ { u_int32_t cacheline; u_int32_t pci_writemax = 0; + u_int32_t hdrsplit; /* Initialize link to down state. */ sc->ti_linkstat = TI_EV_CODE_LINK_DOWN; @@ -1313,16 +1889,23 @@ /* This sets the min dma param all the way up (0xff). */ TI_SETBIT(sc, TI_PCI_STATE, TI_PCISTATE_MINDMA); +#ifdef TI_JUMBO_HDRSPLIT + hdrsplit = TI_OPMODE_JUMBO_HDRSPLIT; +#else + hdrsplit = 0; +#endif + + /* Configure DMA variables. */ #if BYTE_ORDER == BIG_ENDIAN CSR_WRITE_4(sc, TI_GCR_OPMODE, TI_OPMODE_BYTESWAP_BD | TI_OPMODE_BYTESWAP_DATA | TI_OPMODE_WORDSWAP_BD | TI_OPMODE_WARN_ENB | TI_OPMODE_FATAL_ENB | - TI_OPMODE_DONT_FRAG_JUMBO); + TI_OPMODE_DONT_FRAG_JUMBO | hdrsplit); #else CSR_WRITE_4(sc, TI_GCR_OPMODE, TI_OPMODE_BYTESWAP_DATA| TI_OPMODE_WORDSWAP_BD|TI_OPMODE_DONT_FRAG_JUMBO| - TI_OPMODE_WARN_ENB|TI_OPMODE_FATAL_ENB); + TI_OPMODE_WARN_ENB|TI_OPMODE_FATAL_ENB | hdrsplit); #endif /* @@ -1422,8 +2005,15 @@ rcb = &sc->ti_rdata->ti_info.ti_jumbo_rx_rcb; TI_HOSTADDR(rcb->ti_hostaddr) = vtophys(&sc->ti_rdata->ti_rx_jumbo_ring); + +#ifdef PRIVATE_JUMBOS rcb->ti_max_len = TI_JUMBO_FRAMELEN; rcb->ti_flags = 0; +#else + rcb->ti_max_len = PAGE_SIZE; + rcb->ti_flags = TI_RCB_FLAG_USE_EXT_RX_BD; +#endif + if (sc->arpcom.ac_if.if_hwassist) rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM | TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM; @@ -1499,10 +2089,12 @@ vtophys(&sc->ti_tx_considx); /* Set up tuneables */ +#if 0 if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) CSR_WRITE_4(sc, TI_GCR_RX_COAL_TICKS, (sc->ti_rx_coal_ticks / 10)); else +#endif CSR_WRITE_4(sc, TI_GCR_RX_COAL_TICKS, sc->ti_rx_coal_ticks); CSR_WRITE_4(sc, TI_GCR_TX_COAL_TICKS, sc->ti_tx_coal_ticks); CSR_WRITE_4(sc, TI_GCR_STAT_TICKS, sc->ti_stat_ticks); @@ -1543,6 +2135,40 @@ return(ENXIO); } +#ifdef KLD_MODULE +static int +log2rndup(int len) +{ + int log2size = 0, t = len; + while (t > 1) { + log2size++; + t >>= 1; + } + if (len != (1 << log2size)) + log2size++; + return log2size; +} + +static int +ti_mbuf_sanity(device_t dev) +{ + if((mbstat.m_msize != MSIZE) + || mbstat.m_mclbytes != MCLBYTES){ + device_printf(dev, "\n"); + device_printf(dev, + "This module was compiled with -DMCLSHIFT=%d -DMSIZE=%d\n", + MCLSHIFT, MSIZE); + device_printf(dev, + "The kernel was compiled with MCLSHIFT=%d, MSIZE=%d\n", + log2rndup(mbstat.m_mclbytes), (int)mbstat.m_msize); + return EINVAL; + } + return 0; + +} +#endif + + static int ti_attach(dev) device_t dev; { @@ -1554,6 +2180,16 @@ s = splimp(); +#ifdef KLD_MODULE + if(ti_mbuf_sanity(dev)){ + device_printf(dev, "Module mbuf constants do not match kernel constants!\n"); + device_printf(dev, "Rebuild the module or the kernel so they match\n"); + device_printf(dev, "\n"); + error = EINVAL; + goto fail; + } +#endif + sc = device_get_softc(dev); unit = device_get_unit(dev); bzero(sc, sizeof(struct ti_softc)); @@ -1692,6 +2328,7 @@ bzero(sc->ti_rdata, sizeof(struct ti_ring_data)); /* Try to allocate memory for jumbo buffers. */ +#ifdef PRIVATE_JUMBOS if (ti_alloc_jumbo_mem(sc)) { printf("ti%d: jumbo buffer allocation failed\n", sc->ti_unit); bus_teardown_intr(dev, sc->ti_irq, sc->ti_intrhand); @@ -1703,6 +2340,19 @@ error = ENXIO; goto fail; } +#else + if (!jumbo_vm_init()) { + printf("ti%d: VM initialization failed!\n", sc->ti_unit); + bus_teardown_intr(dev, sc->ti_irq, sc->ti_intrhand); + bus_release_resource(dev, SYS_RES_IRQ, 0, sc->ti_irq); + bus_release_resource(dev, SYS_RES_MEMORY, + TI_PCI_LOMEM, sc->ti_res); + contigfree(sc->ti_rdata, sizeof(struct ti_ring_data), + M_DEVBUF); + error = ENOMEM; + goto fail; + } +#endif /* * We really need a better way to tell a 1000baseTX card @@ -1719,12 +2369,19 @@ pci_get_device(dev) == NG_DEVICEID_GA620T) sc->ti_copper = 1; + /* Set default tuneable values. */ sc->ti_stat_ticks = 2 * TI_TICKS_PER_SEC; +#if 0 sc->ti_rx_coal_ticks = TI_TICKS_PER_SEC / 5000; +#endif + sc->ti_rx_coal_ticks = 170; sc->ti_tx_coal_ticks = TI_TICKS_PER_SEC / 500; sc->ti_rx_max_coal_bds = 64; +#if 0 sc->ti_tx_max_coal_bds = 128; +#endif + sc->ti_tx_max_coal_bds = 32; sc->ti_tx_buf_ratio = 21; /* Set up ifnet structure */ @@ -1733,6 +2390,7 @@ ifp->if_unit = sc->ti_unit; ifp->if_name = "ti"; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + tis[unit] = sc; ifp->if_ioctl = ti_ioctl; ifp->if_output = ether_output; ifp->if_start = ti_start; @@ -1774,13 +2432,59 @@ * Call MI attach routine. */ ether_ifattach(ifp, ETHER_BPF_SUPPORTED); + + + bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header)); + /* + * If this is the first card to be initialized, initialize the + * softc queue. + */ + if (unit == 0) + STAILQ_INIT(&ti_sc_list); + + STAILQ_INSERT_TAIL(&ti_sc_list, sc, ti_links); + + /* Register the device */ + sc->dev = make_dev(&ti_cdevsw, sc->ti_unit, UID_ROOT, GID_OPERATOR, + 0600, "ti%d", sc->ti_unit); + fail: splx(s); return(error); } +/* + * Verify that our character special device is not currently + * open. Also track down any cached vnodes & kill them before + * the module is unloaded + */ + +static int +ti_unref_special(device_t dev) +{ + struct vnode *ti_vn; + int count; + struct ti_softc *sc = sc = device_get_softc(dev); + + if(!vfinddev(sc->dev, VCHR, &ti_vn)){ + return 0; + } + + if((count = vcount(ti_vn))){ + device_printf(dev, + "%d refs to special device, denying unload\n", + count); + return count; + } + /* now we know that there's a vnode in the cache. We hunt it + down and kill it now, before unloading */ + vgone(ti_vn); + return 0; +} + + static int ti_detach(dev) device_t dev; { @@ -1788,6 +2492,9 @@ struct ifnet *ifp; int s; + if(ti_unref_special(dev)) + return EBUSY; + s = splimp(); sc = device_get_softc(dev); @@ -1800,7 +2507,11 @@ bus_release_resource(dev, SYS_RES_IRQ, 0, sc->ti_irq); bus_release_resource(dev, SYS_RES_MEMORY, TI_PCI_LOMEM, sc->ti_res); +#ifdef PRIVATE_JUMBOS contigfree(sc->ti_cdata.ti_jumbo_buf, TI_JMEM, M_DEVBUF); +#else +/* ti_free_jumbos(dev);*/ +#endif contigfree(sc->ti_rdata, sizeof(struct ti_ring_data), M_DEVBUF); ifmedia_removeall(&sc->ifmedia); @@ -1809,6 +2520,67 @@ return(0); } +#ifndef PRIVATE_JUMBOS + +/* + * If hdr_len is 0, that means that header splitting wasn't done on + * this packet for some reason. The two most likely reasons are that + * the protocol isn't a supported protocol for splitting, or this + * packet had a fragment offset that wasn't 0. + * + * The header length, if it is non-zero, will always be the length of + * the headers on the packet, but that length could be longer than the + * first mbuf. So we take the minimum of the two as the actual + * length. + */ + + +static /*__inline*/ void +ti_hdr_split(struct mbuf *top, int hdr_len, int pkt_len, int idx) +{ + int i = 0; + int lengths[4] = {0, 0, 0, 0}; + struct mbuf *m, *mp; + + if (hdr_len != 0) + top->m_len = min(hdr_len, top->m_len); + pkt_len -= top->m_len; + lengths[i++] = top->m_len; + + mp = top; + for (m = top->m_next; m && pkt_len; m = m->m_next) { + m->m_len = m->m_ext.ext_size = min(m->m_len, pkt_len); + pkt_len -= m->m_len; + lengths[i++] = m->m_len; + mp = m; + } + +#if 0 + if (hdr_len != 0) + printf("got split packet: "); + else + printf("got non-split packet: "); + + printf("%d,%d,%d,%d = %d\n", lengths[0], + lengths[1], lengths[2], lengths[3], + lengths[0] + lengths[1] + lengths[2] + + lengths[3]); +#endif + + if (pkt_len) + panic("header splitting didn't"); + + if (m) { + m_freem(m); + mp->m_next = NULL; + + } + if (mp->m_next != NULL) + panic("ti_hdr_split: last mbuf in chain should be null"); +} + +#endif + /* * Frame reception handling. This is called if there's a frame * on the receive return list. @@ -1864,6 +2636,16 @@ ti_newbuf_jumbo(sc, sc->ti_jumbo, m); continue; } +#ifdef PRIVATE_JUMBOS + m->m_len = cur_rx->ti_len; +#else /* PRIVATE_JUMBOS */ +#ifdef TI_JUMBO_HDRSPLIT + ti_hdr_split(m, TI_HOSTADDR(cur_rx->ti_addr), + cur_rx->ti_len, rxidx); +#else /* TI_JUMBO_HDRSPLIT */ + m_adj(m, cur_rx->ti_len - m->m_pkthdr.len); +#endif /* TI_JUMBO_HDRSPLIT */ +#endif /* PRIVATE_JUMBOS */ } else if (cur_rx->ti_flags & TI_BDFLAG_MINI_RING) { TI_INC(sc->ti_mini, TI_MINI_RX_RING_CNT); m = sc->ti_cdata.ti_rx_mini_chain[rxidx]; @@ -1878,6 +2660,7 @@ ti_newbuf_mini(sc, sc->ti_mini, m); continue; } + m->m_len = cur_rx->ti_len; } else { TI_INC(sc->ti_std, TI_STD_RX_RING_CNT); m = sc->ti_cdata.ti_rx_std_chain[rxidx]; @@ -1892,9 +2675,10 @@ ti_newbuf_std(sc, sc->ti_std, m); continue; } + m->m_len = cur_rx->ti_len; } - m->m_pkthdr.len = m->m_len = cur_rx->ti_len; + m->m_pkthdr.len = cur_rx->ti_len; ifp->if_ipackets++; eh = mtod(m, struct ether_header *); m->m_pkthdr.rcvif = ifp; @@ -1994,12 +2778,12 @@ sc = xsc; ifp = &sc->arpcom.ac_if; -#ifdef notdef +/*#ifdef notdef*/ /* Avoid this for now -- checking this register is expensive. */ /* Make sure this is really our interrupt. */ if (!(CSR_READ_4(sc, TI_MISC_HOST_CTL) & TI_MHC_INTSTATE)) return; -#endif +/*#endif*/ /* Ack interrupt and stop others from occuring. */ CSR_WRITE_4(sc, TI_MB_HOSTINTR, 1); @@ -2075,6 +2859,7 @@ else if (m_head->m_flags & M_FRAG) csum_flags |= TI_BDFLAG_IP_FRAG; } + /* * Start packing the mbufs in this chain into * the fragment pointers. Stop when we run out @@ -2230,6 +3015,10 @@ splx(s); + /* Register the device */ + sc->dev = make_dev(&ti_cdevsw, sc->ti_unit, UID_ROOT, GID_OPERATOR, + 0600, "ti%d", sc->ti_unit); + return; } @@ -2509,13 +3298,334 @@ return(error); } +static int +ti_open(dev_t dev, int flags, int fmt, struct proc *p) +{ + int unit, s; + struct ti_softc *sc; + + unit = minor(dev) & 0xff; + + sc = ti_lookup_softc(unit); + + if (sc == NULL) + return(ENODEV); + + s = splimp(); + sc->ti_flags |= TI_FLAG_DEBUGING; + splx(s); + + return(0); +} + +static int +ti_close(dev_t dev, int flag, int fmt, struct proc *p) +{ + int unit, s; + struct ti_softc *sc; + + unit = minor(dev) & 0xff; + + sc = ti_lookup_softc(unit); + + if (sc == NULL) + return(ENODEV); + + s = splimp(); + sc->ti_flags &= ~TI_FLAG_DEBUGING; + splx(s); + + return(0); +} + +/* + * This ioctl routine goes along with the Tigon character device. + */ +static int +ti_ioctl2(dev_t dev, u_long cmd, caddr_t addr, int flag, struct proc *p) +{ + int unit, error; + struct ti_softc *sc; + + unit = minor(dev) & 0xff; + + sc = ti_lookup_softc(unit); + + if (sc == NULL) + return(ENODEV); + + error = 0; + + switch(cmd) { + case TIIOCGETSTATS: + { + struct ti_stats *outstats; + + outstats = (struct ti_stats *)addr; + + bcopy(&sc->ti_rdata->ti_info.ti_stats, outstats, + sizeof(struct ti_stats)); + break; + } + case TIIOCGETPARAMS: + { + struct ti_params *params; + + params = (struct ti_params *)addr; + + params->ti_stat_ticks = sc->ti_stat_ticks; + params->ti_rx_coal_ticks = sc->ti_rx_coal_ticks; + params->ti_tx_coal_ticks = sc->ti_tx_coal_ticks; + params->ti_rx_max_coal_bds = sc->ti_rx_max_coal_bds; + params->ti_tx_max_coal_bds = sc->ti_tx_max_coal_bds; + params->ti_tx_buf_ratio = sc->ti_tx_buf_ratio; + params->param_mask = TI_PARAM_ALL; + + error = 0; + + break; + } + case TIIOCSETPARAMS: + { + struct ti_params *params; + + params = (struct ti_params *)addr; + + if (params->param_mask & TI_PARAM_STAT_TICKS) { + sc->ti_stat_ticks = params->ti_stat_ticks; + CSR_WRITE_4(sc, TI_GCR_STAT_TICKS, sc->ti_stat_ticks); + } + + if (params->param_mask & TI_PARAM_RX_COAL_TICKS) { + sc->ti_rx_coal_ticks = params->ti_rx_coal_ticks; + CSR_WRITE_4(sc, TI_GCR_RX_COAL_TICKS, + sc->ti_rx_coal_ticks); + } + + if (params->param_mask & TI_PARAM_TX_COAL_TICKS) { + sc->ti_tx_coal_ticks = params->ti_tx_coal_ticks; + CSR_WRITE_4(sc, TI_GCR_TX_COAL_TICKS, + sc->ti_tx_coal_ticks); + } + + if (params->param_mask & TI_PARAM_RX_COAL_BDS) { + sc->ti_rx_max_coal_bds = params->ti_rx_max_coal_bds; + CSR_WRITE_4(sc, TI_GCR_RX_MAX_COAL_BD, + sc->ti_rx_max_coal_bds); + } + + if (params->param_mask & TI_PARAM_TX_COAL_BDS) { + sc->ti_tx_max_coal_bds = params->ti_tx_max_coal_bds; + CSR_WRITE_4(sc, TI_GCR_TX_MAX_COAL_BD, + sc->ti_tx_max_coal_bds); + } + + if (params->param_mask & TI_PARAM_TX_BUF_RATIO) { + sc->ti_tx_buf_ratio = params->ti_tx_buf_ratio; + CSR_WRITE_4(sc, TI_GCR_TX_BUFFER_RATIO, + sc->ti_tx_buf_ratio); + } + + error = 0; + + break; + } + case TIIOCSETTRACE: { + ti_trace_type trace_type; + + trace_type = *(ti_trace_type *)addr; + + /* + * Set tracing to whatever the user asked for. Setting + * this register to 0 should have the effect of disabling + * tracing. + */ + CSR_WRITE_4(sc, TI_GCR_NIC_TRACING, trace_type); + + error = 0; + + break; + } + case TIIOCGETTRACE: { + struct ti_trace_buf *trace_buf; + u_int32_t trace_start, cur_trace_ptr, trace_len; + + trace_buf = (struct ti_trace_buf *)addr; + + trace_start = CSR_READ_4(sc, TI_GCR_NICTRACE_START); + cur_trace_ptr = CSR_READ_4(sc, TI_GCR_NICTRACE_PTR); + trace_len = CSR_READ_4(sc, TI_GCR_NICTRACE_LEN); + + printf("ti%d: trace_start = %#x, cur_trace_ptr = %#x, " + "trace_len = %d\n", sc->ti_unit, trace_start, + cur_trace_ptr, trace_len); + printf("ti%d: trace_buf->buf_len = %d\n", sc->ti_unit, + trace_buf->buf_len); + /* XXX KDM pick up here */ + + error = ti_copy_mem(sc, trace_start, min(trace_len, + trace_buf->buf_len), + (caddr_t)trace_buf->buf, 1, 1); + + if (error == 0) { + trace_buf->fill_len = min(trace_len, + trace_buf->buf_len); + if (cur_trace_ptr < trace_start) + trace_buf->cur_trace_ptr = + trace_start - cur_trace_ptr; + else + trace_buf->cur_trace_ptr = + cur_trace_ptr - trace_start; + } else + trace_buf->fill_len = 0; + + + break; + } + + /* + * XXX KDM + * For debugging, five ioctls are needed: + * ALT_ATTACH + * ALT_READ_TG_REG + * ALT_WRITE_TG_REG + * ALT_READ_TG_MEM + * ALT_WRITE_TG_MEM + */ + case ALT_ATTACH: + /* + * From what I can tell, Alteon's Solaris Tigon driver + * only has one character device, so you have to attach + * to the Tigon board you're interested in. This seems + * like a not-so-good way to do things, since unless you + * subsequently specify the unit number of the device + * you're interested in in every ioctl, you'll only be + * able to debug one board at a time. + */ + error = 0; + break; + case ALT_READ_TG_MEM: + case ALT_WRITE_TG_MEM: + { + struct tg_mem *mem_param; + u_int32_t sram_end, scratch_end; + + mem_param = (struct tg_mem *)addr; + + if (sc->ti_hwrev == TI_HWREV_TIGON) { + sram_end = TI_END_SRAM_I; + scratch_end = TI_END_SCRATCH_I; + } else { + sram_end = TI_END_SRAM_II; + scratch_end = TI_END_SCRATCH_II; + } + + /* + * For now, we'll only handle accessing regular SRAM, + * nothing else. + */ + if ((mem_param->tgAddr >= TI_BEG_SRAM) + && ((mem_param->tgAddr + mem_param->len) <= sram_end)) { + /* + * In this instance, we always copy to/from user + * space, so the user space argument is set to 1. + */ + error = ti_copy_mem(sc, mem_param->tgAddr, + mem_param->len, + mem_param->userAddr, 1, + (cmd == ALT_READ_TG_MEM) ? 1 : 0); + } else if ((mem_param->tgAddr >= TI_BEG_SCRATCH) + && (mem_param->tgAddr <= scratch_end)) { + error = ti_copy_scratch(sc, mem_param->tgAddr, + mem_param->len, + mem_param->userAddr, 1, + (cmd == ALT_READ_TG_MEM) ? + 1 : 0, TI_PROCESSOR_A); + } else if ((mem_param->tgAddr >= TI_BEG_SCRATCH_B_DEBUG) + && (mem_param->tgAddr <= TI_BEG_SCRATCH_B_DEBUG)) { + if (sc->ti_hwrev == TI_HWREV_TIGON) { + printf("ti%d: invalid memory range for " + "Tigon I\n", sc->ti_unit); + error = EINVAL; + break; + } + error = ti_copy_scratch(sc, mem_param->tgAddr - + TI_SCRATCH_DEBUG_OFF, + mem_param->len, + mem_param->userAddr, 1, + (cmd == ALT_READ_TG_MEM) ? + 1 : 0, TI_PROCESSOR_B); + } else { + printf("ti%d: memory address %#x len %d is out of " + "supported range\n", sc->ti_unit, + mem_param->tgAddr, mem_param->len); + error = EINVAL; + } + + break; + } + case ALT_READ_TG_REG: + case ALT_WRITE_TG_REG: + { + struct tg_reg *regs; + u_int32_t tmpval; + + regs = (struct tg_reg *)addr; + + /* + * Make sure the address in question isn't out of range. + */ + if (regs->addr > TI_REG_MAX) { + error = EINVAL; + break; + } + if (cmd == ALT_READ_TG_REG) { + bus_space_read_region_4(sc->ti_btag, sc->ti_bhandle, + regs->addr, &tmpval, 1); + regs->data = ntohl(tmpval); +#if 0 + if ((regs->addr == TI_CPU_STATE) + || (regs->addr == TI_CPU_CTL_B)) { + printf("ti%d: register %#x = %#x\n", + sc->ti_unit, regs->addr, tmpval); + } +#endif + } else { + tmpval = htonl(regs->data); + bus_space_write_region_4(sc->ti_btag, sc->ti_bhandle, + regs->addr, &tmpval, 1); + } + + break; + } + default: + error = ENOTTY; + break; + } + return(error); +} + + static void ti_watchdog(ifp) struct ifnet *ifp; { + int s; struct ti_softc *sc; sc = ifp->if_softc; + /* + * When we're debugging, the chip is often stopped for long periods + * of time, and that would normally cause the watchdog timer to fire. + * Since that impedes debugging, we don't want to do that. + */ + s = splimp(); + if (sc->ti_flags & TI_FLAG_DEBUGING) { + splx(s); + return; + } + splx(s); + printf("ti%d: watchdog timeout -- resetting\n", sc->ti_unit); ti_stop(sc); ti_init(sc); Index: pci/if_tireg.h =================================================================== RCS file: /home/ncvs/src/sys/pci/if_tireg.h,v retrieving revision 1.13.2.3 diff -u -r1.13.2.3 if_tireg.h --- pci/if_tireg.h 2000/08/24 00:07:59 1.13.2.3 +++ pci/if_tireg.h 2001/05/14 16:37:32 @@ -343,6 +343,7 @@ #define TI_OPMODE_NO_TX_INTRS 0x00002000 #define TI_OPMODE_NO_RX_INTRS 0x00004000 #define TI_OPMODE_FATAL_ENB 0x40000000 /* not yet implimented */ +#define TI_OPMODE_JUMBO_HDRSPLIT 0x00008000 /* * DMA configuration thresholds. @@ -418,9 +419,56 @@ #define TI_MEM_MAX 0x7FFFFF /* - * Even on the alpha, pci addresses are 32-bit quantities + * Maximum register address on the Tigon. */ +#define TI_REG_MAX 0x3fff +/* + * These values were taken from Alteon's tg.h. + */ +#define TI_BEG_SRAM 0x0 /* host thinks it's here */ +#define TI_BEG_SCRATCH 0xc00000 /* beg of scratch pad area */ +#define TI_END_SRAM_II 0x800000 /* end of SRAM, for 2 MB stuffed */ +#define TI_END_SCRATCH_II 0xc04000 /* end of scratch pad CPU A (16KB) */ +#define TI_END_SCRATCH_B 0xc02000 /* end of scratch pad CPU B (8KB) */ +#define TI_BEG_SCRATCH_B_DEBUG 0xd00000 /* beg of scratch pad for ioctl */ +#define TI_END_SCRATCH_B_DEBUG 0xd02000 /* end of scratch pad for ioctl */ +#define TI_SCRATCH_DEBUG_OFF 0x100000 /* offset for ioctl usage */ +#define TI_END_SRAM_I 0x200000 /* end of SRAM, for 2 MB stuffed */ +#define TI_END_SCRATCH_I 0xc00800 /* end of scratch pad area (2KB) */ +#define TI_BEG_PROM 0x40000000 /* beg of PROM, special access */ +#define TI_BEG_FLASH 0x80000000 /* beg of EEPROM, special access */ +#define TI_END_FLASH 0x80100000 /* end of EEPROM for 1 MB stuff */ +#define TI_BEG_SER_EEPROM 0xa0000000 /* beg of Serial EEPROM (fake out) */ +#define TI_END_SER_EEPROM 0xa0002000 /* end of Serial EEPROM (fake out) */ +#define TI_BEG_REGS 0xc0000000 /* beg of register area */ +#define TI_END_REGS 0xc0000400 /* end of register area */ +#define TI_END_WRITE_REGS 0xc0000180 /* can't write GPRs currently */ +#define TI_BEG_REGS2 0xc0000200 /* beg of second writeable reg area */ +/* the EEPROM is byte addressable in a pretty odd way */ +#define EEPROM_BYTE_LOC 0xff000000 + +/* + * From Alteon's tg.h. + */ +#define TI_PROCESSOR_A 0 +#define TI_PROCESSOR_B 1 +#define TI_CPU_A TG_PROCESSOR_A +#define TI_CPU_B TG_PROCESSOR_B + +/* + * Following macro can be used to access to any of the CPU registers + * It will adjust the address appropriately. + * Parameters: + * reg - The register to access, e.g TI_CPU_CONTROL + * cpu - cpu, i.e PROCESSOR_A or PROCESSOR_B (or TI_CPU_A or TI_CPU_B) + */ +#define CPU_REG(reg, cpu) ((reg) + (cpu) * 0x100) + +/* + Even on the alpha, pci addresses are 32-bit quantities + */ + #ifdef __64_bit_pci_addressing__ typedef struct { u_int64_t ti_addr; @@ -482,192 +530,6 @@ }; /* - * Tigon statistics counters. - */ -struct ti_stats { - /* - * MAC stats, taken from RFC 1643, ethernet-like MIB - */ - volatile u_int32_t dot3StatsAlignmentErrors; /* 0 */ - volatile u_int32_t dot3StatsFCSErrors; /* 1 */ - volatile u_int32_t dot3StatsSingleCollisionFrames; /* 2 */ - volatile u_int32_t dot3StatsMultipleCollisionFrames; /* 3 */ - volatile u_int32_t dot3StatsSQETestErrors; /* 4 */ - volatile u_int32_t dot3StatsDeferredTransmissions; /* 5 */ - volatile u_int32_t dot3StatsLateCollisions; /* 6 */ - volatile u_int32_t dot3StatsExcessiveCollisions; /* 7 */ - volatile u_int32_t dot3StatsInternalMacTransmitErrors; /* 8 */ - volatile u_int32_t dot3StatsCarrierSenseErrors; /* 9 */ - volatile u_int32_t dot3StatsFrameTooLongs; /* 10 */ - volatile u_int32_t dot3StatsInternalMacReceiveErrors; /* 11 */ - /* - * interface stats, taken from RFC 1213, MIB-II, interfaces group - */ - volatile u_int32_t ifIndex; /* 12 */ - volatile u_int32_t ifType; /* 13 */ - volatile u_int32_t ifMtu; /* 14 */ - volatile u_int32_t ifSpeed; /* 15 */ - volatile u_int32_t ifAdminStatus; /* 16 */ -#define IF_ADMIN_STATUS_UP 1 -#define IF_ADMIN_STATUS_DOWN 2 -#define IF_ADMIN_STATUS_TESTING 3 - volatile u_int32_t ifOperStatus; /* 17 */ -#define IF_OPER_STATUS_UP 1 -#define IF_OPER_STATUS_DOWN 2 -#define IF_OPER_STATUS_TESTING 3 -#define IF_OPER_STATUS_UNKNOWN 4 -#define IF_OPER_STATUS_DORMANT 5 - volatile u_int32_t ifLastChange; /* 18 */ - volatile u_int32_t ifInDiscards; /* 19 */ - volatile u_int32_t ifInErrors; /* 20 */ - volatile u_int32_t ifInUnknownProtos; /* 21 */ - volatile u_int32_t ifOutDiscards; /* 22 */ - volatile u_int32_t ifOutErrors; /* 23 */ - volatile u_int32_t ifOutQLen; /* deprecated */ /* 24 */ - volatile u_int8_t ifPhysAddress[8]; /* 8 bytes */ /* 25 - 26 */ - volatile u_int8_t ifDescr[32]; /* 27 - 34 */ - u_int32_t alignIt; /* align to 64 bit for u_int64_ts following */ - /* - * more interface stats, taken from RFC 1573, MIB-IIupdate, - * interfaces group - */ - volatile u_int64_t ifHCInOctets; /* 36 - 37 */ - volatile u_int64_t ifHCInUcastPkts; /* 38 - 39 */ - volatile u_int64_t ifHCInMulticastPkts; /* 40 - 41 */ - volatile u_int64_t ifHCInBroadcastPkts; /* 42 - 43 */ - volatile u_int64_t ifHCOutOctets; /* 44 - 45 */ - volatile u_int64_t ifHCOutUcastPkts; /* 46 - 47 */ - volatile u_int64_t ifHCOutMulticastPkts; /* 48 - 49 */ - volatile u_int64_t ifHCOutBroadcastPkts; /* 50 - 51 */ - volatile u_int32_t ifLinkUpDownTrapEnable; /* 52 */ - volatile u_int32_t ifHighSpeed; /* 53 */ - volatile u_int32_t ifPromiscuousMode; /* 54 */ - volatile u_int32_t ifConnectorPresent; /* follow link state 55 */ - /* - * Host Commands - */ - volatile u_int32_t nicCmdsHostState; /* 56 */ - volatile u_int32_t nicCmdsFDRFiltering; /* 57 */ - volatile u_int32_t nicCmdsSetRecvProdIndex; /* 58 */ - volatile u_int32_t nicCmdsUpdateGencommStats; /* 59 */ - volatile u_int32_t nicCmdsResetJumboRing; /* 60 */ - volatile u_int32_t nicCmdsAddMCastAddr; /* 61 */ - volatile u_int32_t nicCmdsDelMCastAddr; /* 62 */ - volatile u_int32_t nicCmdsSetPromiscMode; /* 63 */ - volatile u_int32_t nicCmdsLinkNegotiate; /* 64 */ - volatile u_int32_t nicCmdsSetMACAddr; /* 65 */ - volatile u_int32_t nicCmdsClearProfile; /* 66 */ - volatile u_int32_t nicCmdsSetMulticastMode; /* 67 */ - volatile u_int32_t nicCmdsClearStats; /* 68 */ - volatile u_int32_t nicCmdsSetRecvJumboProdIndex; /* 69 */ - volatile u_int32_t nicCmdsSetRecvMiniProdIndex; /* 70 */ - volatile u_int32_t nicCmdsRefreshStats; /* 71 */ - volatile u_int32_t nicCmdsUnknown; /* 72 */ - /* - * NIC Events - */ - volatile u_int32_t nicEventsNICFirmwareOperational; /* 73 */ - volatile u_int32_t nicEventsStatsUpdated; /* 74 */ - volatile u_int32_t nicEventsLinkStateChanged; /* 75 */ - volatile u_int32_t nicEventsError; /* 76 */ - volatile u_int32_t nicEventsMCastListUpdated; /* 77 */ - volatile u_int32_t nicEventsResetJumboRing; /* 78 */ - /* - * Ring manipulation - */ - volatile u_int32_t nicRingSetSendProdIndex; /* 79 */ - volatile u_int32_t nicRingSetSendConsIndex; /* 80 */ - volatile u_int32_t nicRingSetRecvReturnProdIndex; /* 81 */ - /* - * Interrupts - */ - volatile u_int32_t nicInterrupts; /* 82 */ - volatile u_int32_t nicAvoidedInterrupts; /* 83 */ - /* - * BD Coalessing Thresholds - */ - volatile u_int32_t nicEventThresholdHit; /* 84 */ - volatile u_int32_t nicSendThresholdHit; /* 85 */ - volatile u_int32_t nicRecvThresholdHit; /* 86 */ - /* - * DMA Attentions - */ - volatile u_int32_t nicDmaRdOverrun; /* 87 */ - volatile u_int32_t nicDmaRdUnderrun; /* 88 */ - volatile u_int32_t nicDmaWrOverrun; /* 89 */ - volatile u_int32_t nicDmaWrUnderrun; /* 90 */ - volatile u_int32_t nicDmaWrMasterAborts; /* 91 */ - volatile u_int32_t nicDmaRdMasterAborts; /* 92 */ - /* - * NIC Resources - */ - volatile u_int32_t nicDmaWriteRingFull; /* 93 */ - volatile u_int32_t nicDmaReadRingFull; /* 94 */ - volatile u_int32_t nicEventRingFull; /* 95 */ - volatile u_int32_t nicEventProducerRingFull; /* 96 */ - volatile u_int32_t nicTxMacDescrRingFull; /* 97 */ - volatile u_int32_t nicOutOfTxBufSpaceFrameRetry; /* 98 */ - volatile u_int32_t nicNoMoreWrDMADescriptors; /* 99 */ - volatile u_int32_t nicNoMoreRxBDs; /* 100 */ - volatile u_int32_t nicNoSpaceInReturnRing; /* 101 */ - volatile u_int32_t nicSendBDs; /* current count 102 */ - volatile u_int32_t nicRecvBDs; /* current count 103 */ - volatile u_int32_t nicJumboRecvBDs; /* current count 104 */ - volatile u_int32_t nicMiniRecvBDs; /* current count 105 */ - volatile u_int32_t nicTotalRecvBDs; /* current count 106 */ - volatile u_int32_t nicTotalSendBDs; /* current count 107 */ - volatile u_int32_t nicJumboSpillOver; /* 108 */ - volatile u_int32_t nicSbusHangCleared; /* 109 */ - volatile u_int32_t nicEnqEventDelayed; /* 110 */ - /* - * Stats from MAC rx completion - */ - volatile u_int32_t nicMacRxLateColls; /* 111 */ - volatile u_int32_t nicMacRxLinkLostDuringPkt; /* 112 */ - volatile u_int32_t nicMacRxPhyDecodeErr; /* 113 */ - volatile u_int32_t nicMacRxMacAbort; /* 114 */ - volatile u_int32_t nicMacRxTruncNoResources; /* 115 */ - /* - * Stats from the mac_stats area - */ - volatile u_int32_t nicMacRxDropUla; /* 116 */ - volatile u_int32_t nicMacRxDropMcast; /* 117 */ - volatile u_int32_t nicMacRxFlowControl; /* 118 */ - volatile u_int32_t nicMacRxDropSpace; /* 119 */ - volatile u_int32_t nicMacRxColls; /* 120 */ - /* - * MAC RX Attentions - */ - volatile u_int32_t nicMacRxTotalAttns; /* 121 */ - volatile u_int32_t nicMacRxLinkAttns; /* 122 */ - volatile u_int32_t nicMacRxSyncAttns; /* 123 */ - volatile u_int32_t nicMacRxConfigAttns; /* 124 */ - volatile u_int32_t nicMacReset; /* 125 */ - volatile u_int32_t nicMacRxBufDescrAttns; /* 126 */ - volatile u_int32_t nicMacRxBufAttns; /* 127 */ - volatile u_int32_t nicMacRxZeroFrameCleanup; /* 128 */ - volatile u_int32_t nicMacRxOneFrameCleanup; /* 129 */ - volatile u_int32_t nicMacRxMultipleFrameCleanup; /* 130 */ - volatile u_int32_t nicMacRxTimerCleanup; /* 131 */ - volatile u_int32_t nicMacRxDmaCleanup; /* 132 */ - /* - * Stats from the mac_stats area - */ - volatile u_int32_t nicMacTxCollisionHistogram[15]; /* 133 */ - /* - * MAC TX Attentions - */ - volatile u_int32_t nicMacTxTotalAttns; /* 134 */ - /* - * NIC Profile - */ - volatile u_int32_t nicProfile[32]; /* 135 */ - /* - * Pat to 1024 bytes. - */ - u_int32_t pad[75]; -}; -/* * Tigon general information block. This resides in host memory * and contains the status counters, ring control blocks and * producer pointers. @@ -1035,7 +897,7 @@ #define TI_SSLOTS 256 #define TI_MSLOTS 256 -#define TI_JSLOTS 384 +#define TI_JSLOTS 256 #define TI_JRAWLEN (TI_JUMBO_FRAMELEN + ETHER_ALIGN + sizeof(u_int64_t)) #define TI_JLEN (TI_JRAWLEN + (sizeof(u_int64_t) - \ @@ -1046,7 +908,7 @@ struct ti_jslot { caddr_t ti_buf; - int ti_inuse; + int ti_inuse; }; /* @@ -1057,7 +919,11 @@ */ struct ti_ring_data { struct ti_rx_desc ti_rx_std_ring[TI_STD_RX_RING_CNT]; +#ifdef PRIVATE_JUMBOS struct ti_rx_desc ti_rx_jumbo_ring[TI_JUMBO_RX_RING_CNT]; +#else + struct ti_rx_desc_ext ti_rx_jumbo_ring[TI_JUMBO_RX_RING_CNT]; +#endif struct ti_rx_desc ti_rx_mini_ring[TI_MINI_RX_RING_CNT]; struct ti_rx_desc ti_rx_return_ring[TI_RETURN_RING_CNT]; struct ti_event_desc ti_event_ring[TI_EVENT_RING_CNT]; @@ -1113,7 +979,16 @@ SLIST_ENTRY(ti_jpool_entry) jpool_entries; }; +typedef enum { + TI_FLAG_NONE = 0x00, + TI_FLAG_DEBUGING = 0x01, + TI_FLAG_WAIT_FOR_LINK = 0x02 +} ti_flag_vals; + +#ifdef _KERNEL + struct ti_softc { + STAILQ_ENTRY(ti_softc) ti_links; struct arpcom arpcom; /* interface info */ bus_space_handle_t ti_bhandle; vm_offset_t ti_vhandle; @@ -1149,7 +1024,11 @@ u_int32_t ti_tx_buf_ratio; int ti_if_flags; int ti_txcnt; + ti_flag_vals ti_flags; + dev_t dev; }; + +#endif /* * Microchip Technology 24Cxx EEPROM control bytes Index: pci/ti_fw2.h =================================================================== RCS file: /home/ncvs/src/sys/pci/ti_fw2.h,v retrieving revision 1.6.2.3 diff -u -r1.6.2.3 ti_fw2.h --- pci/ti_fw2.h 2000/08/24 00:07:57 1.6.2.3 +++ pci/ti_fw2.h 2001/05/14 16:14:40 @@ -1,35 +1,29 @@ /* - * Generated by genfw.c - * Built on Wed Jul 26 10:40:31 2000 by wpaul@brak.osd.bsdi.com + * Generated by Ken's special genfw.c + * Built on Thu Jul 20 17:35:07 EDT 2000 by gallatin@grasshopper.cs.duke.edu * OS: FreeBSD 4.0-RELEASE - * - * Note: this is really firmware release 12.4.11 with some minor - * modifications. Release 12.4.13 apparently fails to handle 10/100 - * modes in some cases. - * - * $FreeBSD: src/sys/pci/ti_fw2.h,v 1.6.2.3 2000/08/24 00:07:57 wpaul Exp $ */ static int tigon2FwReleaseMajor = 0xc; static int tigon2FwReleaseMinor = 0x4; static int tigon2FwReleaseFix = 0xd; static u_int32_t tigon2FwStartAddr = 0x00004000; static u_int32_t tigon2FwTextAddr = 0x00004000; -static int tigon2FwTextLen = 0x11c80; -static u_int32_t tigon2FwRodataAddr = 0x00015c80; -static int tigon2FwRodataLen = 0x10d0; -static u_int32_t tigon2FwDataAddr = 0x00016d80; -static int tigon2FwDataLen = 0x1c0; -static u_int32_t tigon2FwSbssAddr = 0x00016f40; -static int tigon2FwSbssLen = 0xcc; -static u_int32_t tigon2FwBssAddr = 0x00017010; -static int tigon2FwBssLen = 0x20c0; +int tigon2FwTextLen = 0x13130; +static u_int32_t tigon2FwRodataAddr = 0x00017130; +int tigon2FwRodataLen = 0x1106; +static u_int32_t tigon2FwDataAddr = 0x00018420; +int tigon2FwDataLen = 0x180; +static u_int32_t tigon2FwSbssAddr = 0x000185a0; +int tigon2FwSbssLen = 0xcc; +static u_int32_t tigon2FwBssAddr = 0x00018670; +int tigon2FwBssLen = 0x20c0; static u_int32_t tigon2FwText[] = { 0x0, 0x10000003, 0x0, 0xd, 0xd, -0x3c1d0001, 0x8fbd6de0, 0x3a0f021, 0x3c100000, -0x26104000, 0xc0010c0, 0x0, 0xd, -0x3c1d0001, 0x8fbd6de4, 0x3a0f021, 0x3c100000, -0x26104000, 0xc0017e0, 0x0, 0xd, +0x3c1d0002, 0x8fbd8460, 0x3a0f021, 0x3c100000, +0x26104000, 0xc001082, 0x0, 0xd, +0x3c1d0002, 0x8fbd8464, 0x3a0f021, 0x3c100000, +0x26104000, 0xc0018cc, 0x0, 0xd, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, @@ -41,470 +35,529 @@ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2000008, -0x0, 0x800172f, 0x3c0a0001, 0x800172f, -0x3c0a0002, 0x800172f, 0x0, 0x8002ca7, -0x0, 0x8002c4a, 0x0, 0x800172f, -0x3c0a0004, 0x8003286, 0x0, 0x8001a52, -0x0, 0x8003948, 0x0, 0x80038ef, -0x0, 0x800172f, 0x3c0a0006, 0x80039b6, -0x3c0a0007, 0x800172f, 0x3c0a0008, 0x800172f, -0x3c0a0009, 0x8003a0e, 0x0, 0x8002ea1, -0x0, 0x800172f, 0x3c0a000b, 0x800172f, -0x3c0a000c, 0x800172f, 0x3c0a000d, 0x80028f7, -0x0, 0x800288c, 0x0, 0x800172f, -0x3c0a000e, 0x800208c, 0x0, 0x8001964, -0x0, 0x8001a04, 0x0, 0x8003ca2, -0x0, 0x8003c90, 0x0, 0x800172f, -0x0, 0x800191a, 0x0, 0x800172f, -0x0, 0x800172f, 0x3c0a0013, 0x800172f, -0x3c0a0014, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x0, -0x0, 0x0, 0x0, 0x27bdffe0, -0x3c1cc000, 0xafbf001c, 0xafb00018, 0x8f820140, -0x24030003, 0xaf8300ec, 0x34420004, 0xc002b1c, -0xaf820140, 0x3c0100c0, 0xc001763, 0xac203ffc, -0x401821, 0x3c020010, 0x3c010001, 0xac236f5c, -0x10620011, 0x43102b, 0x14400002, 0x3c020020, -0x3c020008, 0x1062000c, 0x24050100, 0x3c060001, -0x8cc66f5c, 0x3c040001, 0x24845d34, 0x3821, -0xafa00010, 0xc002b37, 0xafa00014, 0x3c020020, -0x3c010001, 0xac226f5c, 0x24020008, 0x3c010001, -0xac226f74, 0x2402001f, 0x3c010001, 0xac226f84, -0x24020016, 0x3c010001, 0xac226f58, 0x3c05fffe, -0x34a56f08, 0x3c020001, 0x8c426f5c, 0x3c030002, -0x246390d0, 0x3c040001, 0x8c846d84, 0x431023, +0x0, 0x80017d9, 0x3c0a0001, 0x80017d9, +0x3c0a0002, 0x80017d9, 0x0, 0x8002ec4, +0x0, 0x8002e4e, 0x0, 0x80017d9, +0x3c0a0004, 0x80035a3, 0x0, 0x8001b5a, +0x0, 0x8003e2d, 0x0, 0x8003dbb, +0x0, 0x80017d9, 0x3c0a0006, 0x8003eb4, +0x3c0a0007, 0x80017d9, 0x3c0a0008, 0x80017d9, +0x3c0a0009, 0x8003f25, 0x0, 0x80030d9, +0x0, 0x80017d9, 0x3c0a000b, 0x80017d9, +0x3c0a000c, 0x80017d9, 0x3c0a000d, 0x8002af6, +0x0, 0x8002a8a, 0x0, 0x80017d9, +0x3c0a000e, 0x800219b, 0x0, 0x8001a69, +0x0, 0x8001b0b, 0x0, 0x8004203, +0x0, 0x80041f1, 0x0, 0x80017d9, +0x0, 0x8001a1f, 0x0, 0x80017d9, +0x0, 0x80017d9, 0x3c0a0013, 0x80017d9, +0x3c0a0014, 0x27bdffe0, 0x3c1cc000, 0xafbf001c, +0xafb00018, 0x8f820140, 0x24030003, 0xaf8300ec, +0x34420004, 0xaf820140, 0xc002d20, 0x0, +0x3c0100c0, 0xac203ffc, 0xc00184f, 0x0, +0x401821, 0x3c020010, 0x3c010002, 0xac2385bc, +0x10620025, 0x43102b, 0x14400002, 0x3c020020, +0x3c020008, 0x10620020, 0x24050100, 0x3c040001, +0x248471e4, 0x3c060002, 0x8cc685bc, 0x3821, +0xafa00010, 0xc002d3b, 0xafa00014, 0x3c040001, +0x248471f0, 0x24020256, 0xafa20010, 0xafa00014, +0x8f860144, 0x3c070001, 0x24e771f8, 0xc002d3b, +0x3405dead, 0x8f82011c, 0x34420002, 0xaf82011c, +0x8f820220, 0x34420004, 0xaf820220, 0x8f830140, +0x3c020020, 0x3c010002, 0xac2285bc, 0x3c020001, +0x621825, 0xaf830140, 0x24020008, 0x3c010002, +0xac2285d4, 0x2402001f, 0x3c010002, 0xac2285e4, +0x24020016, 0x3c010002, 0xac2285b8, 0x3c05fffe, +0x34a56f08, 0x3c020002, 0x8c4285bc, 0x3c030002, +0x2463a730, 0x3c040002, 0x8c848424, 0x431023, 0x14800002, 0x458021, 0x2610fa38, 0x2402f000, -0x2028024, 0xc001785, 0x2002021, 0x2022823, +0x2028024, 0xc001871, 0x2002021, 0x2022823, 0x3c040020, 0x821823, 0x651823, 0x247bb000, 0x3c03fffe, 0x3463bf08, 0x363b821, 0x3c0600bf, -0x34c6f000, 0x3c070001, 0x8ce76d80, 0x3c0300bf, -0x3463e000, 0x852023, 0x3c010001, 0xac246f68, -0x822023, 0x3c010001, 0xac256f50, 0x52842, -0x3c010001, 0xac226f44, 0x27620ffc, 0x3c010001, -0xac226de0, 0x27621ffc, 0xdb3023, 0x7b1823, -0x3c010001, 0xac246f48, 0x3c010001, 0xac256f6c, -0x3c010001, 0xac226de4, 0xaf860150, 0x10e00011, -0xaf830250, 0x3c1d0001, 0x8fbd6d8c, 0x3a0f021, -0xc001749, 0x0, 0x3c020001, 0x8c426d90, -0x3c030001, 0x8c636d94, 0x2442fe00, 0x24630200, -0x3c010001, 0xac226d90, 0x3c010001, 0x10000004, -0xac236d94, 0x3c1d0001, 0x8fbd6de0, 0x3a0f021, -0x3c020001, 0x8c426d84, 0x1040000d, 0x26fafa38, -0x3c020001, 0x8c426d90, 0x3c030001, 0x8c636d94, -0x3c1a0001, 0x8f5a6d94, 0x2442fa38, 0x246305c8, -0x3c010001, 0xac226d90, 0x3c010001, 0xac236d94, -0x3c020001, 0x8c426d88, 0x14400003, 0x0, -0x3c010001, 0xac206d90, 0xc001151, 0x0, -0x8fbf001c, 0x8fb00018, 0x3e00008, 0x27bd0020, -0x3c020001, 0x8c426d90, 0x3c030001, 0x8c636d94, -0x27bdff98, 0xafb00048, 0x3c100001, 0x8e106778, -0xafb20050, 0x3c120000, 0x26524100, 0xafbf0060, -0xafbe005c, 0xafb50058, 0xafb30054, 0xafb1004c, -0xafa20034, 0xafa30030, 0xafa00010, 0xafa00014, -0x8f860040, 0x3c040001, 0x24845d40, 0x24050200, -0x3c010001, 0xac326f40, 0xc002b37, 0x2003821, -0x8f830040, 0x3c02f000, 0x621824, 0x3c026000, -0x1062000b, 0xa3a0003f, 0x240e0001, 0x3c040001, -0x24845d48, 0xa3ae003f, 0xafa00010, 0xafa00014, -0x8f860040, 0x24050300, 0xc002b37, 0x2003821, -0x8f820240, 0x3c030001, 0x431025, 0xaf820240, -0xaf800048, 0x8f820048, 0x14400005, 0x0, -0xaf800048, 0x8f820048, 0x10400004, 0x0, -0xaf800048, 0x10000003, 0x2e02021, 0xaf80004c, -0x2e02021, 0x3c050001, 0xc002ba4, 0x34a540f8, -0x3402021, 0xc002ba4, 0x240505c8, 0x3c020001, -0x8c426f68, 0x3c0d0001, 0x8dad6f48, 0x3c030001, -0x8c636f44, 0x3c080001, 0x8d086f50, 0x3c090001, -0x8d296f6c, 0x3c0a0001, 0x8d4a6f74, 0x3c0b0001, -0x8d6b6f84, 0x3c0c0001, 0x8d8c6f58, 0x3c040001, -0x24845d54, 0x24050400, 0xaf42013c, 0x8f42013c, -0x24060001, 0x24070001, 0xaf400000, 0xaf4d0138, -0xaf430144, 0xaf480148, 0xaf49014c, 0xaf4a0150, -0xaf4b0154, 0xaf4c0158, 0x2442ff80, 0xaf420140, -0x24020001, 0xafa20010, 0xc002b37, 0xafa00014, -0x8f420138, 0xafa20010, 0x8f42013c, 0xafa20014, -0x8f460144, 0x8f470148, 0x3c040001, 0x24845d60, -0xc002b37, 0x24050500, 0xafb70010, 0xafba0014, -0x8f46014c, 0x8f470150, 0x3c040001, 0x24845d6c, -0xc002b37, 0x24050600, 0x3c020001, 0x8c426f5c, -0x3603821, 0x3c060002, 0x24c690d0, 0x2448ffff, -0x1061824, 0xe81024, 0x43102b, 0x10400006, -0x24050900, 0x3c040001, 0x24845d78, 0xafa80010, -0xc002b37, 0xafa00014, 0x8f82000c, 0xafa20010, -0x8f82003c, 0xafa20014, 0x8f860000, 0x8f870004, -0x3c040001, 0x24845d84, 0xc002b37, 0x24051000, -0x8c020220, 0x8c030224, 0x8c060218, 0x8c07021c, -0x3c040001, 0x24845d8c, 0x24051100, 0xafa20010, -0xc002b37, 0xafa30014, 0xaf800054, 0xaf80011c, -0x8c020218, 0x30420002, 0x10400009, 0x0, -0x8c020220, 0x3c030002, 0x34630004, 0x431025, -0xaf42000c, 0x8c02021c, 0x10000008, 0x34420004, -0x8c020220, 0x3c030002, 0x34630006, 0x431025, -0xaf42000c, 0x8c02021c, 0x34420006, 0xaf420014, -0x8c020218, 0x30420010, 0x1040000a, 0x0, -0x8c02021c, 0x34420004, 0xaf420010, 0x8c020220, -0x3c03000a, 0x34630004, 0x431025, 0x10000009, -0xaf420008, 0x8c020220, 0x3c03000a, 0x34630006, -0x431025, 0xaf420008, 0x8c02021c, 0x34420006, -0xaf420010, 0x24020001, 0xaf8200a0, 0xaf8200b0, -0x8f830054, 0x8f820054, 0xaf8000d0, 0xaf8000c0, -0x10000002, 0x24630064, 0x8f820054, 0x621023, -0x2c420065, 0x1440fffc, 0x0, 0x8c040208, -0x8c05020c, 0x26e20028, 0xaee20020, 0x24020490, -0xaee20010, 0xaee40008, 0xaee5000c, 0x26e40008, -0x8c820000, 0x8c830004, 0xaf820090, 0xaf830094, -0x8c820018, 0xaf8200b4, 0x9482000a, 0xaf82009c, -0x8f420014, 0xaf8200b0, 0x8f8200b0, 0x30420004, -0x1440fffd, 0x0, 0x8f8200b0, 0x3c03ef00, -0x431024, 0x10400021, 0x0, 0x8f8200b4, -0xafa20010, 0x8f820090, 0x8f830094, 0x3c040001, -0x24845d94, 0xafa30014, 0x8f8600b0, 0x8f87009c, -0x3c050001, 0xc002b37, 0x34a5200d, 0x3c040001, -0x24845da0, 0x240203c3, 0xafa20010, 0xafa00014, -0x8f860144, 0x3c070001, 0x24e75da8, 0xc002b37, +0x34c6f000, 0x3c070002, 0x8ce78420, 0x3c0300bf, +0x3463e000, 0x852023, 0x3c010002, 0xac2485c8, +0x822023, 0x3c010002, 0xac2585b0, 0x52842, +0x3c010002, 0xac2285a4, 0x27620ffc, 0x3c010002, +0xac228460, 0x27621ffc, 0xdb3023, 0x7b1823, +0x3c010002, 0xac2485a8, 0x3c010002, 0xac2585cc, +0x3c010002, 0xac228464, 0xaf860150, 0xaf830250, +0x10e00027, 0x33620fff, 0x10400014, 0x2402028b, +0x3c040001, 0x248471f0, 0xafa20010, 0xafa00014, +0x8f860144, 0x3c070001, 0x24e771f8, 0xc002d3b, 0x3405dead, 0x8f82011c, 0x34420002, 0xaf82011c, 0x8f820220, 0x34420004, 0xaf820220, 0x8f820140, -0x3c030001, 0x431025, 0xaf820140, 0x96e20472, -0x96e60452, 0x96e70462, 0xafa20010, 0x96e20482, -0x3c040001, 0x24845db4, 0x24051200, 0xc002b37, -0xafa20014, 0x96f00452, 0x32020001, 0x10400002, -0xb021, 0x24160001, 0x32020002, 0x54400001, -0x36d60002, 0x32020008, 0x54400001, 0x36d60004, -0x32020010, 0x54400001, 0x36d60008, 0x32020020, -0x54400001, 0x36d60010, 0x32020040, 0x54400001, -0x36d60020, 0x32020080, 0x54400001, 0x36d60040, -0x96e60482, 0x30c20200, 0x54400001, 0x36d64000, -0x96e30472, 0x30620200, 0x10400003, 0x30620100, -0x10000003, 0x36d62000, 0x54400001, 0x36d61000, -0x96f00462, 0x32c24000, 0x14400004, 0x3207009b, -0x30c2009b, 0x14e20007, 0x240e0001, 0x32c22000, -0x1440000d, 0x32020001, 0x3062009b, 0x10e20009, -0x240e0001, 0x3c040001, 0x24845dc0, 0x24051300, -0x2003821, 0xa3ae003f, 0xafa30010, 0xc002b37, -0xafa00014, 0x32020001, 0x54400001, 0x36d60080, +0x3c030001, 0x431025, 0xaf820140, 0x3c1d0002, +0x8fbd842c, 0x3a0f021, 0xc001807, 0x0, +0x3c020002, 0x8c428430, 0x3c030002, 0x8c638434, +0x2442fe00, 0x24630200, 0x3c010002, 0xac228430, +0x3c010002, 0xac238434, 0x10000004, 0x0, +0x3c1d0002, 0x8fbd8460, 0x3a0f021, 0x3c020002, +0x8c428424, 0x1040000d, 0x26fafa38, 0x3c020002, +0x8c428430, 0x3c030002, 0x8c638434, 0x3c1a0002, +0x8f5a8434, 0x2442fa38, 0x246305c8, 0x3c010002, +0xac228430, 0x3c010002, 0xac238434, 0x3c020002, +0x8c428428, 0x14400003, 0x0, 0x3c010002, +0xac208430, 0xc001140, 0x0, 0x8fbf001c, +0x8fb00018, 0x3e00008, 0x27bd0020, 0x3c020002, +0x8c428430, 0x3c030002, 0x8c638434, 0x27bdff98, +0xafb00048, 0x3c100001, 0x8e107b94, 0xafb20050, +0x3c120000, 0x26524100, 0xafbf0060, 0xafbe005c, +0xafb50058, 0xafb30054, 0xafb1004c, 0xafa20034, +0xafa30030, 0xafa00010, 0xafa00014, 0x8f860040, +0x3c040001, 0x24847204, 0x24050200, 0x3c010002, +0xac3285a0, 0xc002d3b, 0x2003821, 0x8f830040, +0x3c02f000, 0x621824, 0x3c026000, 0x1062001f, +0xa3a0003f, 0x3c040001, 0x2484720c, 0xafa00010, +0xafa00014, 0x8f860040, 0x24050300, 0xc002d3b, +0x2003821, 0x3c040001, 0x248471f0, 0x240202e1, +0xafa20010, 0xafa00014, 0x8f860144, 0x3c070001, +0x24e771f8, 0xc002d3b, 0x3405dead, 0x8f82011c, +0x34420002, 0xaf82011c, 0x8f820220, 0x34420004, +0xaf820220, 0x8f820140, 0x240e0001, 0x3c030001, +0xa3ae003f, 0x431025, 0xaf820140, 0x8f820240, +0x3c030001, 0x431025, 0xaf820240, 0xaf800048, +0x8f820048, 0x14400005, 0x0, 0xaf800048, +0x8f820048, 0x10400004, 0x0, 0xaf800048, +0x10000003, 0x2e02021, 0xaf80004c, 0x2e02021, +0x3c050001, 0xc002da8, 0x34a540f8, 0x3402021, +0xc002da8, 0x240505c8, 0x3c020002, 0x8c4285c8, +0x3c0d0002, 0x8dad85a8, 0x3c030002, 0x8c6385a4, +0x3c080002, 0x8d0885b0, 0x3c090002, 0x8d2985cc, +0x3c0a0002, 0x8d4a85d4, 0x3c0b0002, 0x8d6b85e4, +0x3c0c0002, 0x8d8c85b8, 0x3c040001, 0x24847218, +0x24050400, 0xaf42013c, 0x8f42013c, 0x24060001, +0x24070001, 0xaf400000, 0xaf4d0138, 0xaf430144, +0xaf480148, 0xaf49014c, 0xaf4a0150, 0xaf4b0154, +0xaf4c0158, 0x2442ff80, 0xaf420140, 0x24020001, +0xafa20010, 0xc002d3b, 0xafa00014, 0x8f420138, +0xafa20010, 0x8f42013c, 0xafa20014, 0x8f460144, +0x8f470148, 0x3c040001, 0x24847224, 0xc002d3b, +0x24050500, 0xafb70010, 0xafba0014, 0x8f46014c, +0x8f470150, 0x3c040001, 0x24847230, 0xc002d3b, +0x24050600, 0x3c020002, 0x8c4285bc, 0x3603821, +0x3c060002, 0x24c6a730, 0x2448ffff, 0x1061824, +0xe81024, 0x43102b, 0x1040001a, 0x24050900, +0x3c040001, 0x2484723c, 0xafa80010, 0xc002d3b, +0xafa00014, 0x3c040001, 0x248471f0, 0x2402033a, +0xafa20010, 0xafa00014, 0x8f860144, 0x3c070001, +0x24e771f8, 0xc002d3b, 0x3405dead, 0x8f82011c, +0x34420002, 0xaf82011c, 0x8f820220, 0x34420004, +0xaf820220, 0x8f820140, 0x3c030001, 0x431025, +0xaf820140, 0x8f82000c, 0xafa20010, 0x8f82003c, +0xafa20014, 0x8f860000, 0x8f870004, 0x3c040001, +0x24847248, 0xc002d3b, 0x24051000, 0x8c020220, +0x8c030224, 0x8c060218, 0x8c07021c, 0x3c040001, +0x24847250, 0x24051100, 0xafa20010, 0xc002d3b, +0xafa30014, 0xaf800054, 0xaf80011c, 0x8c020218, +0x30420002, 0x10400009, 0x0, 0x8c020220, +0x3c030002, 0x34630004, 0x431025, 0xaf42000c, +0x8c02021c, 0x10000008, 0x34420004, 0x8c020220, +0x3c030002, 0x34630006, 0x431025, 0xaf42000c, +0x8c02021c, 0x34420006, 0xaf420014, 0x8c020218, +0x30420010, 0x1040000a, 0x0, 0x8c02021c, +0x34420004, 0xaf420010, 0x8c020220, 0x3c03000a, +0x34630004, 0x431025, 0x10000009, 0xaf420008, +0x8c020220, 0x3c03000a, 0x34630006, 0x431025, +0xaf420008, 0x8c02021c, 0x34420006, 0xaf420010, +0x24020001, 0xaf8200a0, 0xaf8200b0, 0x8f830054, +0x8f820054, 0xaf8000d0, 0xaf8000c0, 0x10000002, +0x24630064, 0x8f820054, 0x621023, 0x2c420065, +0x1440fffc, 0x0, 0x8c040208, 0x8c05020c, +0x26e20028, 0xaee20020, 0x24020490, 0xaee20010, +0xaee40008, 0xaee5000c, 0x26e40008, 0x8c820000, +0x8c830004, 0xaf820090, 0xaf830094, 0x8c820018, +0xaf8200b4, 0x9482000a, 0xaf82009c, 0x8f420014, +0xaf8200b0, 0x8f8200b0, 0x30420004, 0x1440fffd, +0x0, 0x8f8200b0, 0x3c03ef00, 0x431024, +0x10400021, 0x0, 0x8f8200b4, 0xafa20010, +0x8f820090, 0x8f830094, 0x3c040001, 0x24847258, +0xafa30014, 0x8f8600b0, 0x8f87009c, 0x3c050001, +0xc002d3b, 0x34a5200d, 0x3c040001, 0x248471f0, +0x240203c4, 0xafa20010, 0xafa00014, 0x8f860144, +0x3c070001, 0x24e771f8, 0xc002d3b, 0x3405dead, +0x8f82011c, 0x34420002, 0xaf82011c, 0x8f820220, +0x34420004, 0xaf820220, 0x8f820140, 0x3c030001, +0x431025, 0xaf820140, 0x96e20472, 0x96e60452, +0x96e70462, 0xafa20010, 0x96e20482, 0x3c040001, +0x24847264, 0x24051200, 0xc002d3b, 0xafa20014, +0x96f00452, 0x32020001, 0x10400002, 0xb021, +0x24160001, 0x32020002, 0x54400001, 0x36d60002, +0x32020008, 0x54400001, 0x36d60004, 0x32020010, +0x54400001, 0x36d60008, 0x32020020, 0x54400001, +0x36d60010, 0x32020040, 0x54400001, 0x36d60020, +0x32020080, 0x54400001, 0x36d60040, 0x96e60482, +0x30c20200, 0x54400001, 0x36d64000, 0x96e30472, +0x30620200, 0x10400003, 0x30620100, 0x10000003, +0x36d62000, 0x54400001, 0x36d61000, 0x96f00462, +0x32c24000, 0x14400004, 0x3207009b, 0x30c2009b, +0x14e20007, 0x0, 0x32c22000, 0x14400022, +0x32020001, 0x3062009b, 0x10e2001f, 0x32020001, +0x3c040001, 0x24847270, 0x24051300, 0x2003821, +0xafa30010, 0xc002d3b, 0xafa00014, 0x3c040001, +0x248471f0, 0x24020400, 0xafa20010, 0xafa00014, +0x8f860144, 0x3c070001, 0x24e771f8, 0xc002d3b, +0x3405dead, 0x8f82011c, 0x34420002, 0xaf82011c, +0x8f820220, 0x34420004, 0xaf820220, 0x8f820140, +0x240e0001, 0x3c030001, 0xa3ae003f, 0x431025, +0xaf820140, 0x32020001, 0x54400001, 0x36d60080, 0x32020002, 0x54400001, 0x36d60100, 0x32020008, 0x54400001, 0x36d60200, 0x32020010, 0x54400001, 0x36d60400, 0x32020080, 0x54400001, 0x36d60800, 0x8c020218, 0x30420200, 0x10400002, 0x3c020008, -0x2c2b025, 0x8c020218, 0x30420800, 0x10400002, -0x3c020080, 0x2c2b025, 0x8c020218, 0x30420400, -0x10400002, 0x3c020100, 0x2c2b025, 0x8c020218, -0x30420100, 0x10400002, 0x3c020200, 0x2c2b025, -0x8c020218, 0x30420080, 0x10400002, 0x3c020400, -0x2c2b025, 0x8c020218, 0x30422000, 0x10400002, -0x3c020010, 0x2c2b025, 0x8c020218, 0x30424000, -0x10400002, 0x3c020020, 0x2c2b025, 0x8c020218, -0x30421000, 0x10400002, 0x3c020040, 0x2c2b025, -0x8ee20498, 0x8ee3049c, 0xaf420160, 0xaf430164, -0x8ee204a0, 0x8ee304a4, 0xaf420168, 0xaf43016c, -0x8ee204a8, 0x8ee304ac, 0xaf420170, 0xaf430174, -0x8ee20428, 0x8ee3042c, 0xaf420178, 0xaf43017c, -0x8ee20448, 0x8ee3044c, 0xaf420180, 0xaf430184, -0x8ee20458, 0x8ee3045c, 0xaf420188, 0xaf43018c, -0x8ee20468, 0x8ee3046c, 0xaf420190, 0xaf430194, -0x8ee20478, 0x8ee3047c, 0xaf420198, 0xaf43019c, -0x8ee20488, 0x8ee3048c, 0xaf4201a0, 0xaf4301a4, -0x8ee204b0, 0x8ee304b4, 0x24040080, 0xaf4201a8, -0xaf4301ac, 0xc002ba4, 0x24050080, 0x8c02025c, -0x27440224, 0xaf4201f0, 0x8c020260, 0x24050200, -0x24060008, 0xc002bbb, 0xaf4201f8, 0x3c043b9a, -0x3484ca00, 0x3821, 0x24020006, 0x24030002, -0xaf4201f4, 0x240203e8, 0xaf430204, 0xaf430200, -0xaf4401fc, 0xaf420294, 0x24020001, 0xaf430290, -0xaf42029c, 0x3c030001, 0x671821, 0x90636d98, -0x3471021, 0x24e70001, 0xa043022c, 0x2ce2000f, -0x1440fff8, 0x3471821, 0x24e70001, 0x3c080001, -0x350840f8, 0x8f820040, 0x3c040001, 0x24845dcc, -0x24051400, 0x21702, 0x24420030, 0xa062022c, -0x3471021, 0xa040022c, 0x8c070218, 0x2c03021, -0x240205c8, 0xafa20010, 0xc002b37, 0xafa80014, -0x3c040001, 0x24845dd8, 0x3c050000, 0x24a55c80, -0x24060010, 0x27b10030, 0x2203821, 0x27b30034, -0xc0017a3, 0xafb30010, 0x3c030001, 0x8c636d88, -0x1060000a, 0x408021, 0x8fa30030, 0x2405ff00, -0x8fa20034, 0x246400ff, 0x852024, 0x831823, -0x431023, 0xafa20034, 0xafa40030, 0x3c040001, -0x24845de4, 0x3c050000, 0x24a54100, 0x24060108, -0x2203821, 0xc0017a3, 0xafb30010, 0x409021, -0x32c20003, 0x3c010001, 0xac326f40, 0x10400045, -0x2203821, 0x8f820050, 0x3c030010, 0x431024, -0x10400016, 0x0, 0x8c020218, 0x30420040, -0x1040000f, 0x24020001, 0x8f820050, 0x8c030218, -0x240e0001, 0x3c040001, 0x24845df0, 0xa3ae003f, +0x2c2b025, 0x8c020218, 0x30428000, 0x10400002, +0x3c021000, 0x2c2b025, 0x8c020218, 0x30420800, +0x10400002, 0x3c020080, 0x2c2b025, 0x8c020218, +0x30420400, 0x10400002, 0x3c020100, 0x2c2b025, +0x8c020218, 0x30420100, 0x10400002, 0x3c020200, +0x2c2b025, 0x8c020218, 0x30420080, 0x10400002, +0x3c020400, 0x2c2b025, 0x8c020218, 0x30422000, +0x10400002, 0x3c020010, 0x2c2b025, 0x8c020218, +0x30424000, 0x10400002, 0x3c020020, 0x2c2b025, +0x8c020218, 0x30421000, 0x10400002, 0x3c020040, +0x2c2b025, 0x8ee20498, 0x8ee3049c, 0xaf420160, +0xaf430164, 0x8ee204a0, 0x8ee304a4, 0xaf420168, +0xaf43016c, 0x8ee204a8, 0x8ee304ac, 0xaf420170, +0xaf430174, 0x8ee20428, 0x8ee3042c, 0xaf420178, +0xaf43017c, 0x8ee20448, 0x8ee3044c, 0xaf420180, +0xaf430184, 0x8ee20458, 0x8ee3045c, 0xaf420188, +0xaf43018c, 0x8ee20468, 0x8ee3046c, 0xaf420190, +0xaf430194, 0x8ee20478, 0x8ee3047c, 0xaf420198, +0xaf43019c, 0x8ee20488, 0x8ee3048c, 0xaf4201a0, +0xaf4301a4, 0x8ee204b0, 0x8ee304b4, 0x24040080, +0xaf4201a8, 0xaf4301ac, 0xc002da8, 0x24050080, +0x8c02025c, 0x27440224, 0xaf4201f0, 0x8c020260, +0x24050200, 0x24060008, 0xaf4201f8, 0xc002dbf, +0x0, 0x3c043b9a, 0x3484ca00, 0x3821, +0x24020006, 0x24030002, 0xaf4201f